Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-4046: Add FtpRemoteFileTemplate.ExistsMode

Browse Source 
JIRA: https://jira.spring.io/browse/INT-4046

Since not all FTP servers provide proper `STAT` command implementation,
plus the `NLIST` doesn't work properly for directories cases, introduce the `FtpRemoteFileTemplate.ExistsMode`
to let:
* to perform `STAT` by default (previous) behavior;
* to switch to `NLIST` for `FtpRemoteFileTemplate` internal use;
* perform the full `NLIST` and `FTPClient.changeWorkingDirectory()` algorithm if needed.

* Improve (S)Ftp components to use proper `RemoteFileTemplate` for internal instantiation
* Introduce `FtpMessageHandler` to wrap `FtpRemoteFileTemplate` with the proper `NLIST` `ExistsMode`
* Cover `NLIST` switching from the `FtpOutboundChannelAdapterParser` and `FtpOutboundGatewayParser`
* Document the `FtpRemoteFileTemplate.ExistsMode`

* Fix typo in the recently introduced `RemoteFileOperations.getSession()` method name
* Add JavaDoc to `Session.exists()`
* Add `NLIST` support for the `FtpSession.exists()` to meet the API requirements

**Cherry-pick to 4.2.x and 4.1.x**

Addressing PR comments

Doc Polishing

Conflicts:
	spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java
	spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java
	spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java
	spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpSession.java
	spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParserTests.java
	spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java
	spring-integration-sftp/src/main/java/org/springframework/integration/sftp/outbound/SftpMessageHandler.java
	spring-integration-sftp/src/main/java/org/springframework/integration/sftp/outbound/package-info.java
This commit is contained in:
Artem Bilan
2016-06-06 12:25:26 -04:00
parent f6d6fc9e4e
commit 80e1a56653
23 changed files with 327 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParser.java
View File

@@ -15,22 +15,46 @@
*/
package org.springframework.integration.ftp.config;
import org.w3c.dom.Element;
import org.springframework.beans.factory.config.BeanDefinition;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.integration.file.config.RemoteFileOutboundChannelAdapterParser;
import org.springframework.integration.file.remote.RemoteFileOperations;
import org.springframework.integration.ftp.outbound.FtpMessageHandler;
import org.springframework.integration.ftp.session.FtpRemoteFileTemplate;
/**
* Parser for FTP Outbound Channel Adapters.
*
* @author Gary Russell
* @author Artem Bilan
* @since 4.1
*
*/
public class FtpOutboundChannelAdapterParser extends RemoteFileOutboundChannelAdapterParser {
@Override
protected Class<?> handlerClass() {
return FtpMessageHandler.class;
}
@Override
protected Class<? extends RemoteFileOperations<?>> getTemplateClass() {
return FtpRemoteFileTemplate.class;
}
@Override
protected void postProcessBuilder(BeanDefinitionBuilder builder, Element element) {
BeanDefinition templateDefinition = (BeanDefinition) builder.getRawBeanDefinition()
.getConstructorArgumentValues()
.getIndexedArgumentValues()
.values()
.iterator()
.next()
.getValue();
templateDefinition.getPropertyValues()
.add("existsMode", FtpRemoteFileTemplate.ExistsMode.NLST);
}
}

17
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParser.java
View File

@@ -15,6 +15,10 @@
*/
package org.springframework.integration.ftp.config;
import org.w3c.dom.Element;
import org.springframework.beans.factory.config.BeanDefinition;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.integration.file.config.AbstractRemoteFileOutboundGatewayParser;
import org.springframework.integration.file.remote.RemoteFileOperations;
import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter;
@@ -50,4 +54,17 @@ public class FtpOutboundGatewayParser extends AbstractRemoteFileOutboundGatewayP
return FtpRemoteFileTemplate.class;
}
@Override
protected void postProcessBuilder(BeanDefinitionBuilder builder, Element element) {
BeanDefinition templateDefinition = (BeanDefinition) builder.getRawBeanDefinition()
.getConstructorArgumentValues()
.getIndexedArgumentValues()
.values()
.iterator()
.next()
.getValue();
templateDefinition.getPropertyValues()
.add("existsMode", FtpRemoteFileTemplate.ExistsMode.NLST);
}
}

7
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java
View File

@@ -28,6 +28,7 @@ import org.springframework.integration.file.remote.RemoteFileTemplate;
import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.ftp.session.FtpFileInfo;
import org.springframework.integration.ftp.session.FtpRemoteFileTemplate;
/**
* Outbound Gateway for performing remote file operations via FTP/FTPS.
@@ -46,7 +47,8 @@ public class FtpOutboundGateway extends AbstractRemoteFileOutboundGateway<FTPFil
*/
public FtpOutboundGateway(SessionFactory<FTPFile> sessionFactory,
MessageSessionCallback<FTPFile, ?> messageSessionCallback) {
super(sessionFactory, messageSessionCallback);
this(new FtpRemoteFileTemplate(sessionFactory), messageSessionCallback);
((FtpRemoteFileTemplate) this.remoteFileTemplate).setExistsMode(FtpRemoteFileTemplate.ExistsMode.NLST);
}
/**
@@ -68,7 +70,8 @@ public class FtpOutboundGateway extends AbstractRemoteFileOutboundGateway<FTPFil
* @param expression the filename expression.
*/
public FtpOutboundGateway(SessionFactory<FTPFile> sessionFactory, String command, String expression) {
super(sessionFactory, command, expression);
this(new FtpRemoteFileTemplate(sessionFactory), command, expression);
((FtpRemoteFileTemplate) this.remoteFileTemplate).setExistsMode(FtpRemoteFileTemplate.ExistsMode.NLST);
}
/**

50
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/outbound/FtpMessageHandler.java Normal file
View File

@@ -0,0 +1,50 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.ftp.outbound;
import org.apache.commons.net.ftp.FTPFile;
import org.springframework.integration.file.remote.RemoteFileTemplate;
import org.springframework.integration.file.remote.handler.FileTransferringMessageHandler;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.file.support.FileExistsMode;
import org.springframework.integration.ftp.session.FtpRemoteFileTemplate;
/**
* The FTP specific {@link FileTransferringMessageHandler} extension.
* Based on the {@link FtpRemoteFileTemplate}.
*
* @author Artem Bilan
* @since 4.1.9
* @see FtpRemoteFileTemplate
*/
public class FtpMessageHandler extends FileTransferringMessageHandler<FTPFile> {
public FtpMessageHandler(SessionFactory<FTPFile> sessionFactory) {
this(new FtpRemoteFileTemplate(sessionFactory));
((FtpRemoteFileTemplate) this.remoteFileTemplate).setExistsMode(FtpRemoteFileTemplate.ExistsMode.NLST);
}
public FtpMessageHandler(FtpRemoteFileTemplate remoteFileTemplate) {
super(remoteFileTemplate);
}
public FtpMessageHandler(RemoteFileTemplate<FTPFile> remoteFileTemplate, FileExistsMode mode) {
super(remoteFileTemplate, mode);
}
}

4
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/outbound/package-info.java Normal file
View File

@@ -0,0 +1,4 @@
/**
* Provides classes for the FTP outbound channel adapter.
*/
package org.springframework.integration.ftp.outbound;

79
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpRemoteFileTemplate.java
View File

@@ -26,17 +26,22 @@ import org.springframework.integration.file.remote.SessionCallback;
import org.springframework.integration.file.remote.session.Session;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.messaging.MessagingException;
import org.springframework.util.Assert;
import org.springframework.util.ObjectUtils;
/**
* FTP version of {@code RemoteFileTemplate} providing type-safe access to
* the underlying FTPClient object.
*
* @author Gary Russell
* @author Artem Bilan
* @since 4.1
*
*/
public class FtpRemoteFileTemplate extends RemoteFileTemplate<FTPFile> {
private ExistsMode existsMode = ExistsMode.STAT;
public FtpRemoteFileTemplate(SessionFactory<FTPFile> sessionFactory) {
super(sessionFactory);
}
@@ -47,6 +52,19 @@ public class FtpRemoteFileTemplate extends RemoteFileTemplate<FTPFile> {
return doExecuteWithClient((ClientCallback<FTPClient, T>) callback);
}
/**
* Specify an {@link ExistsMode} for {@link #exists(String)} operation.
* Defaults to {@link ExistsMode#STAT}.
* When used internally by framework components for file operation,
* switched to {@link ExistsMode#NLST}.
* @param existsMode the {@link ExistsMode} to use.
* @since 4.1.9
*/
public void setExistsMode(ExistsMode existsMode) {
Assert.notNull(existsMode, "'existsMode' must not be null.");
this.existsMode = existsMode;
}
protected <T> T doExecuteWithClient(final ClientCallback<FTPClient, T> callback) {
return execute(new SessionCallback<FTPFile, T>() {
@@ -57,21 +75,76 @@ public class FtpRemoteFileTemplate extends RemoteFileTemplate<FTPFile> {
});
}
/**
* This particular FTP implementation is based on the {@link FTPClient#getStatus(String)}
* by default, but since not all FTP servers properly implement the {@code STAT} command,
* the framework internal {@link FtpRemoteFileTemplate} instances are switched to the
* {@link FTPClient#listNames(String)} for only files operations.
* <p> The mode can be switched with the {@link #setExistsMode(ExistsMode)} property.
* <p> Any custom implementation can be done in an extension of the {@link FtpRemoteFileTemplate}.
* @param path the remote file path to check.
* @return true or false if remote file exists or not.
*/
@Override
public boolean exists(final String path) {
return executeWithClient(new ClientCallback<FTPClient, Boolean>() {
return doExecuteWithClient(new ClientCallback<FTPClient, Boolean>() {
@Override
public Boolean doWithClient(FTPClient client) {
try {
return client.getStatus(path) != null;
switch (FtpRemoteFileTemplate.this.existsMode) {
case STAT:
return client.getStatus(path) != null;
case NLST:
String[] names = client.listNames(path);
return !ObjectUtils.isEmpty(names);
case NLST_AND_DIRS:
return FtpRemoteFileTemplate.this.sessionFactory.getSession().exists(path);
default:
throw new IllegalStateException("Unsupported 'existsMode': " +
FtpRemoteFileTemplate.this.existsMode);
}
}
catch (IOException e) {
throw new MessagingException("Failed to stat " + path, e);
throw new MessagingException("Failed to check the remote path for " + path, e);
}
}
});
}
/**
* The {@link #exists(String)} operation mode.
* @since 4.1.9
*/
public enum ExistsMode {
/**
* Perform the {@code STAT} FTP command.
* Default.
*/
STAT,
/**
* Perform the {@code NLST} FTP command.
* Used as default internally by framework components for files only operations.
*/
NLST,
/**
* Perform the {@code NLST} FTP command and fall back to
* {@link FTPClient#changeWorkingDirectory(String)}.
* <p> This technique is required when you want to check if a directory exists
* and the server does not support {@code STAT} - it requires 4 requests/replies.
* <p> If you are only checking for an existing file, {@code NLST} is preferred
* (unless {@code STAT} is supported).
* @see FtpSession#exists(String)
*/
NLST_AND_DIRS
}
}

23
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpSession.java
View File

@@ -29,6 +29,7 @@ import org.apache.commons.net.ftp.FTPReply;
import org.springframework.integration.file.remote.session.Session;
import org.springframework.util.Assert;
import org.springframework.util.ObjectUtils;
/**
* Implementation of {@link Session} for FTP.
@@ -195,17 +196,21 @@ public class FtpSession implements Session<FTPFile> {
public boolean exists(String path) throws IOException{
Assert.hasText(path, "'path' must not be empty");
String currentWorkingPath = this.client.printWorkingDirectory();
Assert.state(currentWorkingPath != null, "working directory cannot be determined, therefore exists check can not be completed");
boolean exists = false;
String[] names = this.client.listNames(path);
boolean exists = !ObjectUtils.isEmpty(names);
try {
if (this.client.changeWorkingDirectory(path)) {
exists = true;
if (!exists) {
String currentWorkingPath = this.client.printWorkingDirectory();
Assert.state(currentWorkingPath != null,
"working directory cannot be determined; exists check can not be completed");
try {
exists = this.client.changeWorkingDirectory(path);
}
}
finally {
this.client.changeWorkingDirectory(currentWorkingPath);
finally {
this.client.changeWorkingDirectory(currentWorkingPath);
}
}
return exists;

8
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParserTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2014 the original author or authors.
* Copyright 2002-2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -38,6 +38,7 @@ import org.springframework.integration.file.remote.handler.FileTransferringMessa
import org.springframework.integration.file.remote.session.CachingSessionFactory;
import org.springframework.integration.file.support.FileExistsMode;
import org.springframework.integration.ftp.session.DefaultFtpSessionFactory;
import org.springframework.integration.ftp.session.FtpRemoteFileTemplate;
import org.springframework.integration.handler.ExpressionEvaluatingMessageProcessor;
import org.springframework.integration.handler.advice.AbstractRequestHandlerAdvice;
import org.springframework.integration.support.MessageBuilder;
@@ -91,7 +92,8 @@ public class FtpOutboundChannelAdapterParserTests {
assertEquals(ftpChannel, TestUtils.getPropertyValue(ftpOutbound, "inputChannel"));
assertEquals("ftpOutbound", ftpOutbound.getComponentName());
FileTransferringMessageHandler<?> handler = TestUtils.getPropertyValue(ftpOutbound, "handler", FileTransferringMessageHandler.class);
String remoteFileSeparator = (String) TestUtils.getPropertyValue(handler, "remoteFileTemplate.remoteFileSeparator");
String remoteFileSeparator = (String) TestUtils.getPropertyValue(handler,
"remoteFileTemplate.remoteFileSeparator");
assertNotNull(remoteFileSeparator);
assertEquals(".foo", TestUtils.getPropertyValue(handler, "remoteFileTemplate.temporaryFileSuffix", String.class));
assertEquals("", remoteFileSeparator);
@@ -99,6 +101,8 @@ public class FtpOutboundChannelAdapterParserTests {
assertEquals("UTF-8", TestUtils.getPropertyValue(handler, "remoteFileTemplate.charset"));
assertNotNull(TestUtils.getPropertyValue(handler, "remoteFileTemplate.directoryExpressionProcessor"));
assertNotNull(TestUtils.getPropertyValue(handler, "remoteFileTemplate.temporaryDirectoryExpressionProcessor"));
assertEquals(FtpRemoteFileTemplate.ExistsMode.NLST,
TestUtils.getPropertyValue(handler, "remoteFileTemplate.existsMode"));
Object sfProperty = TestUtils.getPropertyValue(handler, "remoteFileTemplate.sessionFactory");
assertEquals(DefaultFtpSessionFactory.class, sfProperty.getClass());
DefaultFtpSessionFactory sessionFactory = (DefaultFtpSessionFactory) sfProperty;

13
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java
View File

@@ -41,6 +41,7 @@ import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOut
import org.springframework.integration.file.remote.session.CachingSessionFactory;
import org.springframework.integration.file.support.FileExistsMode;
import org.springframework.integration.ftp.gateway.FtpOutboundGateway;
import org.springframework.integration.ftp.session.FtpRemoteFileTemplate;
import org.springframework.integration.handler.ExpressionEvaluatingMessageProcessor;
import org.springframework.integration.handler.advice.AbstractRequestHandlerAdvice;
import org.springframework.integration.support.MessageBuilder;
@@ -93,7 +94,7 @@ public class FtpOutboundGatewayParserTests {
assertNotNull(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory"));
assertNotNull(TestUtils.getPropertyValue(gateway, "outputChannel"));
assertEquals("local-test-dir", TestUtils.getPropertyValue(gateway, "localDirectoryExpression.literalValue"));
assertFalse((Boolean) TestUtils.getPropertyValue(gateway, "autoCreateLocalDirectory"));
assertFalse(TestUtils.getPropertyValue(gateway, "autoCreateLocalDirectory", Boolean.class));
assertNotNull(TestUtils.getPropertyValue(gateway, "filter"));
assertEquals(Command.LS, TestUtils.getPropertyValue(gateway, "command"));
@@ -105,7 +106,8 @@ public class FtpOutboundGatewayParserTests {
Long sendTimeout = TestUtils.getPropertyValue(gateway, "messagingTemplate.sendTimeout", Long.class);
assertEquals(Long.valueOf(777), sendTimeout);
assertTrue(TestUtils.getPropertyValue(gateway, "requiresReply", Boolean.class));
assertThat(TestUtils.getPropertyValue(gateway, "mputFilter"), Matchers.instanceOf(RegexPatternFileListFilter.class));
assertThat(TestUtils.getPropertyValue(gateway, "mputFilter"),
Matchers.instanceOf(RegexPatternFileListFilter.class));
assertEquals(FileExistsMode.APPEND, TestUtils.getPropertyValue(gateway, "fileExistsMode"));
}
@@ -115,10 +117,13 @@ public class FtpOutboundGatewayParserTests {
"handler", FtpOutboundGateway.class);
assertEquals("X", TestUtils.getPropertyValue(gateway, "remoteFileTemplate.remoteFileSeparator"));
assertNotNull(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory"));
assertTrue(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory") instanceof CachingSessionFactory);
assertThat(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory"),
Matchers.instanceOf(CachingSessionFactory.class));
assertEquals(FtpRemoteFileTemplate.ExistsMode.NLST,
TestUtils.getPropertyValue(gateway, "remoteFileTemplate.existsMode"));
assertNotNull(TestUtils.getPropertyValue(gateway, "outputChannel"));
assertEquals("local-test-dir", TestUtils.getPropertyValue(gateway, "localDirectoryExpression.literalValue"));
assertFalse((Boolean) TestUtils.getPropertyValue(gateway, "autoCreateLocalDirectory"));
assertFalse(TestUtils.getPropertyValue(gateway, "autoCreateLocalDirectory", Boolean.class));
assertEquals(Command.GET, TestUtils.getPropertyValue(gateway, "command"));
@SuppressWarnings("unchecked")
Set<String> options = TestUtils.getPropertyValue(gateway, "options", Set.class);

3
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java
View File

@@ -348,13 +348,14 @@ public class FtpServerOutboundTests {
@Test
public void testInt3412FileMode() {
FtpRemoteFileTemplate template = new FtpRemoteFileTemplate(ftpSessionFactory);
assertFalse(template.exists("ftpTarget/appending.txt"));
Message<String> m = MessageBuilder.withPayload("foo")
.setHeader(FileHeaders.FILENAME, "appending.txt")
.build();
appending.send(m);
appending.send(m);
FtpRemoteFileTemplate template = new FtpRemoteFileTemplate(ftpSessionFactory);
assertLength6(template);
ignoring.send(m);

4
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/session/FtpRemoteFileTemplateTests.java
View File

@@ -71,7 +71,7 @@ public class FtpRemoteFileTemplateTests {
}
@Test
public void testINT3412AppendStatRmdir() {
public void testINT3412AppendStatRmdir() throws IOException {
FtpRemoteFileTemplate template = new FtpRemoteFileTemplate(sessionFactory);
DefaultFileNameGenerator fileNameGenerator = new DefaultFileNameGenerator();
fileNameGenerator.setExpression("'foobar.txt'");
@@ -114,7 +114,7 @@ public class FtpRemoteFileTemplateTests {
assertTrue(session.rmdir("foo/"));
}
});
assertFalse(template.exists("foo"));
assertFalse(sessionFactory.getSession().exists("foo"));
}
@Test