Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-3088 (S)FTP Outbound Gateway - PUT and MPUT

Browse Source 
- Core support in file module
- FTP Parser and Test

https://jira.springsource.org/browse/INT-3088

INT-3088 Add SFTP Support for PUT, MPUT

INT-3088 Polishing - PR Comments

INT-3088 Docbook For PUT/MPUT
This commit is contained in:
Gary Russell
2013-11-19 12:56:07 +02:00
committed by Artem Bilan
parent 88de8117aa
commit 59d6f7dfc1
29 changed files with 1163 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java
View File

@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.commons.net.ftp.FTPFile;
import org.springframework.integration.file.remote.AbstractFileInfo;
import org.springframework.integration.file.remote.RemoteFileTemplate;
import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.ftp.session.FtpFileInfo;
@@ -40,6 +41,11 @@ public class FtpOutboundGateway extends AbstractRemoteFileOutboundGateway<FTPFil
super(sessionFactory, command, expression);
}
private FtpOutboundGateway(RemoteFileTemplate<FTPFile> remoteFileTemplate, String command, String expression) {
super(remoteFileTemplate, command, expression);
}
@Override
protected boolean isDirectory(FTPFile file) {
return file.isDirectory();

120
spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-3.0.xsd
View File

@@ -26,73 +26,6 @@
<xsd:all>
<xsd:element name="request-handler-advice-chain" type="integration:adviceChainType" minOccurs="0" maxOccurs="1" />
</xsd:all>
<xsd:attribute name="remote-directory-expression"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide a SpEL expression which
will compute the directory
path where the files will be transferred to
(e.g., "headers.['remote_dir'] +
'/myTransfers'");
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="temporary-remote-directory-expression"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide a SpEL expression which
will compute the temporary directory
path where files will be transferred to before they are moved to the remote-directory
(e.g., "headers.['remote_dir'] +
'/temp/myTransfers'");
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="auto-create-directory" type="xsd:string"
default="false">
<xsd:annotation>
<xsd:documentation>
Specify whether to automatically create the
remote target directory if
it doesn't exist.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="remote-filename-generator" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to specify a reference to
[org.springframework.integration.file.FileNameGenerator] bean.
</xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type
type="org.springframework.integration.file.FileNameGenerator" />
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="remote-filename-generator-expression"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide SpEL expression which
will compute file name of
the remote file (e.g., assuming payload
is java.io.File
"payload.getName() + '.transfered'");
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="use-temporary-file-name" type="xsd:string" default="true">
<xsd:annotation>
<xsd:documentation>
Allows you to suppress using a temporary file name while writing the file.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="order" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
@@ -113,6 +46,7 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attributeGroup ref="int-file:remoteOutboundAttributeGroup" />
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
@@ -268,7 +202,7 @@
</xsd:annotation>
<xsd:complexType>
<xsd:complexContent>
<xsd:extension base="base-adapter-type">
<xsd:extension base="base-ftp-adapter-type">
<xsd:all>
<xsd:element name="request-handler-advice-chain" type="integration:adviceChainType" minOccurs="0" maxOccurs="1" />
</xsd:all>
@@ -379,7 +313,9 @@
<xsd:documentation>
Allows you to specify a reference to
[org.springframework.integration.file.filters.FileListFilter]
bean.
bean. This filter acts against the remote server view when using the 'ls'
or 'mget' commands.
Only one of 'filter', 'filename-pattern', or 'filename-regex' is allowed.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
@@ -387,10 +323,11 @@
<xsd:annotation>
<xsd:documentation>
Allows you to provide file name pattern to
determine the file names retrieved by the ls command
determine the file names retrieved by the 'ls' and 'mget' commands
and is based
on simple pattern matching algorithm (e.g., "*.txt, fo*.txt"
etc.)
Only one of 'filter', 'filename-pattern', or 'filename-regex' is allowed.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
@@ -398,9 +335,49 @@
<xsd:annotation>
<xsd:documentation>
Allows you to provide Regular Expression to
determine the file names retrieved by the ls command.
determine the file names retrieved by the 'ls' and 'mget' commands.
(e.g.,
"f[o]+\.txt" etc.)
Only one of 'filter', 'filename-pattern', or 'filename-regex' is allowed.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="mput-filter" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type
type="org.springframework.integration.file.filters.FileListFilter" />
</tool:annotation>
</xsd:appinfo>
<xsd:documentation>
Allows you to specify a reference to
[org.springframework.integration.file.filters.FileListFilter]
bean. This filter acts on the local file system when using the 'mput' command.
Only one of 'mput-filter', 'mput-pattern', or 'mput-regex' is allowed.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="mput-pattern" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide file name pattern to
determine the file names sent by the 'mput' command
and is based
on simple pattern matching algorithm (e.g., "*.txt, fo*.txt"
etc.)
Only one of 'mput-filter', 'mput-pattern', or 'mput-regex' is allowed.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="mput-regex" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide Regular Expression to
determine the file names sent by the 'mput' command
(e.g.,
"f[o]+\.txt" etc.)
Only one of 'mput-filter', 'mput-pattern', or 'mput-regex' is allowed.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
@@ -486,6 +463,7 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attributeGroup ref="int-file:remoteOutboundAttributeGroup" />
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>

10
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/TesFtpServer.java
View File

@@ -101,7 +101,7 @@ public class TesFtpServer {
fos.close();
targetFtpDirectory = new File(ftpRootFolder, "ftpTarget");
targetFtpDirectory.mkdirs();
targetFtpDirectory.mkdir();
}
};
this.localFolder = new TemporaryFolder() {
@@ -123,7 +123,7 @@ public class TesFtpServer {
file.createNewFile();
targetLocalDirectory = new File(rootFolder, "localTarget");
targetLocalDirectory.mkdirs();
targetLocalDirectory.mkdir();
}
};
}
@@ -183,14 +183,16 @@ public class TesFtpServer {
}
public static void recursiveDelete(File file) {
public void recursiveDelete(File file) {
File[] files = file.listFiles();
if (files != null) {
for (File each : files) {
recursiveDelete(each);
}
}
file.delete();
if (!(file.equals(this.targetFtpDirectory) || file.equals(this.targetLocalDirectory))) {
file.delete();
}
}

22
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests-context.xml
View File

@@ -29,6 +29,7 @@
command-options="-1 -f"
expression="payload"
order="1"
mput-regex=".*"
/>
<bean id="fooString" class="java.lang.String">
@@ -49,6 +50,7 @@
order="2"
requires-reply="false"
local-filename-generator-expression="#remoteFileName.toUpperCase() + '.a' + @fooString"
mput-pattern="*"
>
<int-ftp:request-handler-advice-chain>
<bean class="org.springframework.integration.ftp.config.FtpOutboundGatewayParserTests$FooAdvice" />
@@ -66,6 +68,26 @@
order="1"
/>
<int-ftp:outbound-gateway id="gateway4"
session-factory="csf"
request-channel="inbound1"
reply-channel="outbound"
command="mput"
expression="payload"
remote-directory="/foo"
remote-file-separator="X"
auto-create-directory="true"
remote-filename-generator="fileNameGenerator"
temporary-remote-directory="/bar"
rename-expression="'foo'"
order="1"
mput-regex=".*"
/>
<bean id="fileNameGenerator" class="org.mockito.Mockito" factory-method="mock">
<constructor-arg value="org.springframework.integration.file.FileNameGenerator"/>
</bean>
<int-ftp:outbound-gateway id="withBeanExpression"
local-directory="local-test-dir"
session-factory="sf"

37
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java
View File

@@ -18,18 +18,25 @@ package org.springframework.integration.ftp.config;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import java.lang.reflect.Method;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import org.hamcrest.Matchers;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.expression.Expression;
import org.springframework.integration.Message;
import org.springframework.integration.endpoint.AbstractEndpoint;
import org.springframework.integration.file.FileNameGenerator;
import org.springframework.integration.file.filters.RegexPatternFileListFilter;
import org.springframework.integration.file.filters.SimplePatternFileListFilter;
import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway.Command;
import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway.Option;
import org.springframework.integration.file.remote.session.CachingSessionFactory;
@@ -64,16 +71,22 @@ public class FtpOutboundGatewayParserTests {
@Autowired
AbstractEndpoint gateway3;
@Autowired
AbstractEndpoint gateway4;
@Autowired
AbstractEndpoint withBeanExpression;
@Autowired
FileNameGenerator generator;
private static volatile int adviceCalled;
@Test
public void testGateway1() {
FtpOutboundGateway gateway = TestUtils.getPropertyValue(gateway1,
"handler", FtpOutboundGateway.class);
assertEquals("X", TestUtils.getPropertyValue(gateway, "remoteFileSeparator"));
assertEquals("X", TestUtils.getPropertyValue(gateway, "remoteFileTemplate.remoteFileSeparator"));
assertNotNull(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory"));
assertNotNull(TestUtils.getPropertyValue(gateway, "outputChannel"));
assertEquals("local-test-dir", TestUtils.getPropertyValue(gateway, "localDirectoryExpression.literalValue"));
@@ -89,13 +102,14 @@ public class FtpOutboundGatewayParserTests {
Long sendTimeout = TestUtils.getPropertyValue(gateway, "messagingTemplate.sendTimeout", Long.class);
assertEquals(Long.valueOf(777), sendTimeout);
assertTrue(TestUtils.getPropertyValue(gateway, "requiresReply", Boolean.class));
assertThat(TestUtils.getPropertyValue(gateway, "mputFilter"), Matchers.instanceOf(RegexPatternFileListFilter.class));
}
@Test
public void testGateway2() throws Exception {
FtpOutboundGateway gateway = TestUtils.getPropertyValue(gateway2,
"handler", FtpOutboundGateway.class);
assertEquals("X", TestUtils.getPropertyValue(gateway, "remoteFileSeparator"));
assertEquals("X", TestUtils.getPropertyValue(gateway, "remoteFileTemplate.remoteFileSeparator"));
assertNotNull(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory"));
assertTrue(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory") instanceof CachingSessionFactory);
assertNotNull(TestUtils.getPropertyValue(gateway, "outputChannel"));
@@ -123,6 +137,7 @@ public class FtpOutboundGatewayParserTests {
}
});
assertEquals("FOO.afoo", genMethod.get().invoke(gateway, new GenericMessage<String>(""), "foo"));
assertThat(TestUtils.getPropertyValue(gateway, "mputFilter"), Matchers.instanceOf(SimplePatternFileListFilter.class));
}
@Test
@@ -135,6 +150,24 @@ public class FtpOutboundGatewayParserTests {
assertEquals("'foo'", TestUtils.getPropertyValue(gateway, "renameProcessor.expression.expression"));
}
@Test
public void testGatewayMPut() {
FtpOutboundGateway gateway = TestUtils.getPropertyValue(gateway4,
"handler", FtpOutboundGateway.class);
assertNotNull(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory"));
assertNotNull(TestUtils.getPropertyValue(gateway, "outputChannel"));
assertEquals(Command.MPUT, TestUtils.getPropertyValue(gateway, "command"));
assertEquals("'foo'", TestUtils.getPropertyValue(gateway, "renameProcessor.expression.expression"));
assertThat(TestUtils.getPropertyValue(gateway, "mputFilter"), Matchers.instanceOf(RegexPatternFileListFilter.class));
assertSame(generator, TestUtils.getPropertyValue(gateway, "remoteFileTemplate.fileNameGenerator"));
assertEquals("/foo",
TestUtils.getPropertyValue(gateway, "remoteFileTemplate.directoryExpressionProcessor.expression", Expression.class)
.getExpressionString());
assertEquals("/bar",
TestUtils.getPropertyValue(gateway, "remoteFileTemplate.temporaryDirectoryExpressionProcessor.expression", Expression.class)
.getExpressionString());
}
@Test
public void testWithBeanExpression() {
FtpOutboundGateway gateway = TestUtils.getPropertyValue(withBeanExpression,

36
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml
View File

@@ -68,4 +68,40 @@
local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"
reply-channel="output"/>
<int:channel id="inboundMPut"/>
<int-ftp:outbound-gateway session-factory="ftpSessionFactory"
request-channel="inboundMPut"
command="mput"
auto-create-directory="true"
filename-pattern="*.txt"
expression="payload"
remote-directory="ftpTarget"
reply-channel="output"/>
<int:channel id="inboundMPutRecursive"/>
<int-ftp:outbound-gateway session-factory="ftpSessionFactory"
request-channel="inboundMPutRecursive"
command="mput"
command-options="-R"
auto-create-directory="true"
filename-pattern="*.txt"
expression="payload"
remote-directory="ftpTarget"
reply-channel="output"/>
<int:channel id="inboundMPutRecursiveFiltered"/>
<int-ftp:outbound-gateway session-factory="ftpSessionFactory"
request-channel="inboundMPutRecursiveFiltered"
command="mput"
command-options="-R"
mput-regex="(.*1.txt|sub.*)"
auto-create-directory="true"
filename-pattern="*.txt"
expression="payload"
remote-directory="ftpTarget"
reply-channel="output"/>
</beans>

74
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java
View File

@@ -16,6 +16,9 @@
package org.springframework.integration.ftp.outbound;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
@@ -85,10 +88,19 @@ public class FtpServerOutboundTests {
@Autowired
private DirectChannel inboundMGetRecursiveFiltered;
@Autowired
private DirectChannel inboundMPut;
@Autowired
private DirectChannel inboundMPutRecursive;
@Autowired
private DirectChannel inboundMPutRecursiveFiltered;
@Before
public void setup() {
TesFtpServer.recursiveDelete(ftpServer.getTargetLocalDirectory());
TesFtpServer.recursiveDelete(ftpServer.getTargetFtpDirectory());
this.ftpServer.recursiveDelete(ftpServer.getTargetLocalDirectory());
this.ftpServer.recursiveDelete(ftpServer.getTargetFtpDirectory());
}
@Test
@@ -231,5 +243,63 @@ public class FtpServerOutboundTests {
assertEquals("source2", new String(baos2.toByteArray()));
}
@Test
public void testInt3088MPutNotRecursive() {
this.inboundMPut.send(new GenericMessage<File>(this.ftpServer.getSourceLocalDirectory()));
@SuppressWarnings("unchecked")
Message<List<String>> out = (Message<List<String>>) this.output.receive(1000);
assertNotNull(out);
assertEquals(2, out.getPayload().size());
assertThat(out.getPayload().get(0),
not(equalTo(out.getPayload().get(1))));
assertThat(
out.getPayload().get(0),
anyOf(equalTo("ftpTarget/localSource1.txt"), equalTo("ftpTarget/localSource2.txt")));
assertThat(
out.getPayload().get(1),
anyOf(equalTo("ftpTarget/localSource1.txt"), equalTo("ftpTarget/localSource2.txt")));
}
@Test
public void testInt3088MPutRecursive() {
this.inboundMPutRecursive.send(new GenericMessage<File>(this.ftpServer.getSourceLocalDirectory()));
@SuppressWarnings("unchecked")
Message<List<String>> out = (Message<List<String>>) this.output.receive(1000);
assertNotNull(out);
assertEquals(3, out.getPayload().size());
assertThat(out.getPayload().get(0),
not(equalTo(out.getPayload().get(1))));
assertThat(
out.getPayload().get(0),
anyOf(equalTo("ftpTarget/localSource1.txt"), equalTo("ftpTarget/localSource2.txt"),
equalTo("ftpTarget/subLocalSource/subLocalSource1.txt")));
assertThat(
out.getPayload().get(1),
anyOf(equalTo("ftpTarget/localSource1.txt"), equalTo("ftpTarget/localSource2.txt"),
equalTo("ftpTarget/subLocalSource/subLocalSource1.txt")));
assertThat(
out.getPayload().get(2),
anyOf(equalTo("ftpTarget/localSource1.txt"), equalTo("ftpTarget/localSource2.txt"),
equalTo("ftpTarget/subLocalSource/subLocalSource1.txt")));
}
@Test
public void testInt3088MPutRecursiveFiltered() {
this.inboundMPutRecursiveFiltered.send(new GenericMessage<File>(this.ftpServer.getSourceLocalDirectory()));
@SuppressWarnings("unchecked")
Message<List<String>> out = (Message<List<String>>) this.output.receive(1000);
assertNotNull(out);
assertEquals(2, out.getPayload().size());
assertThat(out.getPayload().get(0),
not(equalTo(out.getPayload().get(1))));
assertThat(
out.getPayload().get(0),
anyOf(equalTo("ftpTarget/localSource1.txt"), equalTo("ftpTarget/localSource2.txt"),
equalTo("ftpTarget/subLocalSource/subLocalSource1.txt")));
assertThat(
out.getPayload().get(1),
anyOf(equalTo("ftpTarget/localSource1.txt"), equalTo("ftpTarget/localSource2.txt"),
equalTo("ftpTarget/subLocalSource/subLocalSource1.txt")));
}
}