Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-4142: Backport Streaming (S)FTP

Browse Source 
JIRA: https://jira.spring.io/browse/INT-4142

INT-4015: Streaming Remote File Inbound Adapter

Initial commit.

Reworked to emit an input stream and use the file splitter.

Add StreamTransformer.

Add CLOSABLE_RESOURCE header so we can close the session automatically.

Implement INT-3854, FTP, SFTP

(S)FTP Namespace Changes

Docs - also fixes a PDF overflow

Polishing - PR Comments

checkstyle fixes

Polishing - Add Namespace for StreamParser

Polishing - PR Comments

Fix Streaming (S)FTP Tests

https://build.spring.io/browse/INT-B43-190/

Add `AcceptOneFileListFilter`s.

Polishing

polishing

Polishing

* Polish `.travis.yml` to avoid unnecessary work on Travis CI
This commit is contained in:
Gary Russell
2016-04-27 15:10:02 -04:00
committed by Artem Bilan
parent 3bd15cf7c0
commit 5b80ff499c
39 changed files with 1751 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
spring-integration-sftp/src/main/java/org/springframework/integration/sftp/config/SftpInboundChannelAdapterParser.java
View File

@@ -17,6 +17,8 @@
package org.springframework.integration.sftp.config;
import org.springframework.integration.file.config.AbstractRemoteFileInboundChannelAdapterParser;
import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.file.remote.synchronizer.InboundFileSynchronizer;
import org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter;
import org.springframework.integration.sftp.filters.SftpSimplePatternFileListFilter;
import org.springframework.integration.sftp.inbound.SftpInboundFileSynchronizer;
@@ -37,18 +39,18 @@ public class SftpInboundChannelAdapterParser extends AbstractRemoteFileInboundCh
}
@Override
protected String getInboundFileSynchronizerClassname() {
return SftpInboundFileSynchronizer.class.getName();
protected Class<? extends InboundFileSynchronizer> getInboundFileSynchronizerClass() {
return SftpInboundFileSynchronizer.class;
}
@Override
protected String getSimplePatternFileListFilterClassname() {
return SftpSimplePatternFileListFilter.class.getName();
protected Class<? extends FileListFilter<?>> getSimplePatternFileListFilterClass() {
return SftpSimplePatternFileListFilter.class;
}
@Override
protected String getRegexPatternFileListFilterClassname() {
return SftpRegexPatternFileListFilter.class.getName();
protected Class<? extends FileListFilter<?>> getRegexPatternFileListFilterClass() {
return SftpRegexPatternFileListFilter.class;
}
}

1
spring-integration-sftp/src/main/java/org/springframework/integration/sftp/config/SftpNamespaceHandler.java
View File

@@ -32,6 +32,7 @@ public class SftpNamespaceHandler extends AbstractIntegrationNamespaceHandler {
@Override
public void init() {
registerBeanDefinitionParser("inbound-channel-adapter", new SftpInboundChannelAdapterParser());
registerBeanDefinitionParser("inbound-streaming-channel-adapter", new SftpStreamingInboundChannelAdapterParser());
registerBeanDefinitionParser("outbound-channel-adapter", new SftpOutboundChannelAdapterParser());
registerBeanDefinitionParser("outbound-gateway", new SftpOutboundGatewayParser());
}

55
spring-integration-sftp/src/main/java/org/springframework/integration/sftp/config/SftpStreamingInboundChannelAdapterParser.java Normal file
View File

@@ -0,0 +1,55 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.sftp.config;
import org.springframework.integration.core.MessageSource;
import org.springframework.integration.file.config.AbstractRemoteFileStreamingInboundChannelAdapterParser;
import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.file.remote.RemoteFileOperations;
import org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter;
import org.springframework.integration.sftp.filters.SftpSimplePatternFileListFilter;
import org.springframework.integration.sftp.inbound.SftpStreamingMessageSource;
import org.springframework.integration.sftp.session.SftpRemoteFileTemplate;
/**
* @author Gary Russell
* @since 4.2.12
*
*/
public class SftpStreamingInboundChannelAdapterParser extends AbstractRemoteFileStreamingInboundChannelAdapterParser {
@Override
protected Class<? extends RemoteFileOperations<?>> getTemplateClass() {
return SftpRemoteFileTemplate.class;
}
@Override
protected Class<? extends MessageSource<?>> getMessageSourceClass() {
return SftpStreamingMessageSource.class;
}
@Override
protected Class<? extends FileListFilter<?>> getSimplePatternFileListFilterClass() {
return SftpSimplePatternFileListFilter.class;
}
@Override
protected Class<? extends FileListFilter<?>> getRegexPatternFileListFilterClass() {
return SftpRegexPatternFileListFilter.class;
}
}

74
spring-integration-sftp/src/main/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSource.java Normal file
View File

@@ -0,0 +1,74 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.sftp.inbound;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import org.springframework.integration.file.remote.AbstractFileInfo;
import org.springframework.integration.file.remote.AbstractRemoteFileStreamingMessageSource;
import org.springframework.integration.file.remote.RemoteFileTemplate;
import org.springframework.integration.sftp.session.SftpFileInfo;
import com.jcraft.jsch.ChannelSftp.LsEntry;
/**
* Message source for streaming SFTP remote file contents.
*
* @author Gary Russell
* @since 4.2.12
*
*/
public class SftpStreamingMessageSource extends AbstractRemoteFileStreamingMessageSource<LsEntry> {
/**
* Construct an instance with the supplied template.
* @param template the template.
*/
public SftpStreamingMessageSource(RemoteFileTemplate<LsEntry> template) {
super(template, null);
}
/**
* Construct an instance with the supplied template and comparator.
* Note: the comparator is applied each time the remote directory is listed
* which only occurs when the previous list is exhausted.
* @param template the template.
* @param comparator the comparator.
*/
public SftpStreamingMessageSource(RemoteFileTemplate<LsEntry> template,
Comparator<AbstractFileInfo<LsEntry>> comparator) {
super(template, comparator);
}
@Override
public String getComponentType() {
return "sftp:inbound-streaming-channel-adapter";
}
@Override
protected List<AbstractFileInfo<LsEntry>> asFileInfoList(Collection<LsEntry> files) {
List<AbstractFileInfo<LsEntry>> canonicalFiles = new ArrayList<AbstractFileInfo<LsEntry>>();
for (LsEntry file : files) {
canonicalFiles.add(new SftpFileInfo(file));
}
return canonicalFiles;
}
}

237
spring-integration-sftp/src/main/resources/org/springframework/integration/sftp/config/spring-integration-sftp-4.2.xsd
View File

@@ -22,7 +22,7 @@
</xsd:annotation>
<xsd:complexType>
<xsd:complexContent>
<xsd:extension base="base-sftp-adapter-type">
<xsd:extension base="base-outbound-adapter-type">
<xsd:all>
<xsd:element ref="integration:poller" minOccurs="0" maxOccurs="1" />
<xsd:element name="request-handler-advice-chain" type="integration:handlerAdviceChainType"
@@ -71,68 +71,16 @@
<xsd:element name="inbound-channel-adapter">
<xsd:annotation>
<xsd:documentation><![CDATA[
Builds an inbound-channel-adapter that synchronizes with a remote SFTP endpoint.
]]></xsd:documentation>
<xsd:documentation>
Configures a 'SourcePollingChannelAdapter' Endpoint for the
'org.springframework.integration.sftp.inbound.SftpInboundFileSynchronizingMessageSource'
that synchronizes with a remote SFTP endpoint.
</xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:complexContent>
<xsd:extension base="base-sftp-adapter-type">
<xsd:sequence>
<xsd:element ref="integration:poller" minOccurs="0"
maxOccurs="1" />
</xsd:sequence>
<xsd:attribute name="channel" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.messaging.MessageChannel"/>
</tool:annotation>
</xsd:appinfo>
<xsd:documentation>
Identifies channel attached to this adapter. This channel where messages will be sent
to by this adapter.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="comparator" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[
Specify a Comparator to be used when ordering Files. If none is provided, the
order will be determined by the java.io.File implementation of Comparable.
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="filter" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type
type="org.springframework.integration.file.filters.FileListFilter" />
</tool:annotation>
</xsd:appinfo>
<xsd:documentation>
Allows you to specify a reference to a
[org.springframework.integration.file.filters.FileListFilter]
bean. This filter is applied to files on the remote server and
only files that pass the filter are retrieved.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="filename-pattern" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide a file name pattern to
determine the file names
that need to be scanned.
This is based on
simple pattern matching (e.g., "*.txt, fo*.txt"
etc.)
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="local-filename-generator-expression"
type="xsd:string">
<xsd:extension base="base-inbound-adapter-type">
<xsd:attribute name="local-filename-generator-expression" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide a SpEL expression to
@@ -148,15 +96,12 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="filename-regex" type="xsd:string">
<xsd:attribute name="comparator" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide a Regular Expression to
determine the file names
that need to be scanned.
(e.g.,
"f[o]+\.txt" etc.)
</xsd:documentation>
<xsd:documentation><![CDATA[
Specify a Comparator to be used when ordering Files. If none is provided, the
order will be determined by the java.io.File implementation of Comparable.
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="local-filter" type="xsd:string">
@@ -219,16 +164,29 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="remote-directory-expression"
type="xsd:string">
<xsd:attributeGroup ref="tempSuffixGroup" />
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
</xsd:element>
<xsd:element name="inbound-streaming-channel-adapter">
<xsd:annotation>
<xsd:documentation>
Configures a 'SourcePollingChannelAdapter' Endpoint for the
'org.springframework.integration.ftp.inbound.FtpInboundStreamingMessageSource'.
</xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:complexContent>
<xsd:extension base="base-inbound-adapter-type">
<xsd:attribute name="comparator" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Specify a SpEL expression which
will be used to evaluate the directory
path from where the files will be transferred
(e.g., "@someBean.fetchDirectory").
Mutually exclusive with 'remote-directory'.
</xsd:documentation>
<xsd:documentation><![CDATA[
Specify a Comparator to be used when ordering Files. If none is provided, the
order in which files are processed is the order they are received from the
SFTP server. The generic type of the Comparator must be 'SftpFileInfo'.
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
</xsd:extension>
@@ -244,7 +202,7 @@
</xsd:annotation>
<xsd:complexType>
<xsd:complexContent>
<xsd:extension base="base-sftp-adapter-type">
<xsd:extension base="base-outbound-adapter-type">
<xsd:all>
<xsd:element ref="integration:poller" minOccurs="0" maxOccurs="1" />
<xsd:element name="request-handler-advice-chain" type="integration:handlerAdviceChainType"
@@ -479,8 +437,7 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="auto-create-local-directory"
type="xsd:boolean">
<xsd:attribute name="auto-create-local-directory" type="xsd:boolean">
<xsd:annotation>
<xsd:documentation>
Tells this adapter if local directory must be
@@ -524,15 +481,63 @@
</xsd:complexType>
</xsd:element>
<xsd:complexType name="base-sftp-adapter-type">
<xsd:complexType name="base-inbound-adapter-type">
<xsd:complexContent>
<xsd:extension base="base-adapter-type">
<xsd:attribute name="remote-directory" type="xsd:string" use="optional">
<xsd:sequence>
<xsd:element ref="integration:poller" minOccurs="0"
maxOccurs="1" />
</xsd:sequence>
<xsd:attribute name="channel" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.messaging.MessageChannel"/>
</tool:annotation>
</xsd:appinfo>
<xsd:documentation>
Identifies channel attached to this adapter.
The channel to which messages will be sent
by this adapter.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="filename-pattern" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Identifies the directory path (e.g.,
"/temp/mytransfers")
Mutually exclusive with 'remote-directory-expression'.
Allows you to provide a file name pattern to
determine the file names
that need to be scanned.
This is based on
simple pattern matching (e.g., "*.txt, fo*.txt"
etc.)
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="filename-regex" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide a Regular Expression to
determine the file names
that need to be scanned.
(e.g.,
"f[o]+\.txt" etc.)
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="filter" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type
type="org.springframework.integration.file.filters.FileListFilter" />
</tool:annotation>
</xsd:appinfo>
<xsd:documentation>
Allows you to specify a reference to a
[org.springframework.integration.file.filters.FileListFilter]
bean. This filter is applied to files on the remote server and
only files that pass the filter are retrieved.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
@@ -540,7 +545,7 @@
use="optional">
<xsd:annotation>
<xsd:documentation>
Identifies the remote temporary directory path (e.g., "/remote/temp/mytransfers")
Not used.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
@@ -557,10 +562,24 @@
</xsd:complexContent>
</xsd:complexType>
<xsd:complexType name="base-outbound-adapter-type">
<xsd:complexContent>
<xsd:extension base="base-adapter-type">
<xsd:attribute name="temporary-remote-directory" type="xsd:string" use="optional">
<xsd:annotation>
<xsd:documentation>
Identifies the remote temporary directory path (e.g., "/remote/temp/mytransfers")
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attributeGroup ref="tempSuffixGroup" />
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:complexType name="base-adapter-type">
<xsd:attribute name="id" type="xsd:string" />
<xsd:attribute name="session-factory" type="xsd:string"
use="required">
<xsd:attribute name="session-factory" type="xsd:string" use="required">
<xsd:annotation>
<xsd:appinfo>
<tool:annotation kind="ref">
@@ -574,16 +593,6 @@
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="temporary-file-suffix" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Extension used when downloading files. We
change
it right after we know it's
downloaded.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="remote-file-separator" type="xsd:string"
default="/">
<xsd:annotation>
@@ -594,7 +603,39 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="remote-directory" type="xsd:string" use="optional">
<xsd:annotation>
<xsd:documentation>
Identifies the remote directory path (e.g., "/remote/mytransfers")
Mutually exclusive with 'remote-directory-expression'.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="remote-directory-expression"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Specify a SpEL expression which
will be used to evaluate the directory
path to where the files will be transferred
(e.g., "headers.['remote_dir'] + '/myTransfers'" for outbound endpoints)
There is no root object (message) for inbound endpoints
(e.g., "@someBean.fetchDirectory");
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attributeGroup ref="integration:smartLifeCycleAttributeGroup"/>
</xsd:complexType>
<xsd:attributeGroup name="tempSuffixGroup">
<xsd:attribute name="temporary-file-suffix" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Extension used when downloading files. We change
it right after we know it's downloaded.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
</xsd:attributeGroup>
</xsd:schema>

97
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/SftpTestSupport.java Normal file
View File

@@ -0,0 +1,97 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.sftp;
import java.io.File;
import java.util.Collections;
import org.apache.sshd.SshServer;
import org.apache.sshd.common.NamedFactory;
import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
import org.apache.sshd.server.Command;
import org.apache.sshd.server.PasswordAuthenticator;
import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
import org.apache.sshd.server.sftp.SftpSubsystem;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.springframework.integration.file.remote.RemoteFileTestSupport;
import org.springframework.integration.file.remote.session.CachingSessionFactory;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
import com.jcraft.jsch.ChannelSftp.LsEntry;
/**
* Provides an embedded SFTP Server for test cases.
*
* @author David Turanski
* @author Gary Russell
* @since 4.2.12
*/
public class SftpTestSupport extends RemoteFileTestSupport {
private static SshServer server;
public String getTargetLocalDirectoryName() {
return targetLocalDirectory.getAbsolutePath() + File.separator;
}
@Override
public String prefix() {
return "sftp";
}
@BeforeClass
public static void createServer() throws Exception {
server = SshServer.setUpDefaultServer();
server.setPasswordAuthenticator(new PasswordAuthenticator() {
@Override
public boolean authenticate(String username, String password,
org.apache.sshd.server.session.ServerSession session) {
return true;
}
});
server.setPort(0);
server.setKeyPairProvider(new SimpleGeneratorHostKeyProvider("hostkey.ser"));
server.setSubsystemFactories(Collections.<NamedFactory<Command>>singletonList(new SftpSubsystem.Factory()));
server.setFileSystemFactory(new VirtualFileSystemFactory(remoteTemporaryFolder.getRoot().getAbsolutePath()));
server.start();
port = server.getPort();
}
public static SessionFactory<LsEntry> sessionFactory() {
DefaultSftpSessionFactory factory = new DefaultSftpSessionFactory(true);
factory.setHost("localhost");
factory.setPort(port);
factory.setUser("foo");
factory.setPassword("foo");
factory.setAllowUnknownKeys(true);
return new CachingSessionFactory<LsEntry>(factory);
}
@AfterClass
public static void stopServer() throws Exception {
server.stop();
File hostkey = new File("hostkey.ser");
if (hostkey.exists()) {
hostkey.delete();
}
}
}

48
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpStreamingInboundChannelAdapterParserTests-context.xml Normal file
View File

@@ -0,0 +1,48 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-sftp="http://www.springframework.org/schema/integration/sftp"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/integration/sftp http://www.springframework.org/schema/integration/sftp/spring-integration-sftp.xsd">
<bean id="sftpSessionFactory"
class="org.springframework.integration.sftp.config.SftpStreamingInboundChannelAdapterParserTests$TestSessionFactoryBean"/>
<bean id="csf" class="org.springframework.integration.file.remote.session.CachingSessionFactory">
<constructor-arg ref="sftpSessionFactory"/>
</bean>
<int-sftp:inbound-streaming-channel-adapter id="sftpInbound"
channel="sftpChannel"
session-factory="csf"
auto-startup="false"
phase="23"
filename-pattern="*.txt"
remote-file-separator="X"
comparator="comparator"
remote-directory-expression="'foo/bar'">
<int:poller fixed-rate="1000" />
</int-sftp:inbound-streaming-channel-adapter>
<int:channel id="sftpChannel">
<int:queue/>
</int:channel>
<bean id="comparator" class="org.mockito.Mockito" factory-method="mock">
<constructor-arg value="java.util.Comparator"/>
</bean>
<int-sftp:inbound-streaming-channel-adapter id="contextLoadsWithNoComparator"
channel="sftpChannel"
session-factory="csf"
auto-startup="false"
phase="23"
filename-pattern="*.txt"
remote-file-separator="X"
remote-directory-expression="'foo/bar'">
<int:poller fixed-rate="1000" />
</int-sftp:inbound-streaming-channel-adapter>
</beans>

99
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpStreamingInboundChannelAdapterParserTests.java Normal file
View File

@@ -0,0 +1,99 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.sftp.config;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.integration.endpoint.SourcePollingChannelAdapter;
import org.springframework.integration.file.remote.session.CachingSessionFactory;
import org.springframework.integration.sftp.filters.SftpSimplePatternFileListFilter;
import org.springframework.integration.sftp.inbound.SftpStreamingMessageSource;
import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
import org.springframework.integration.sftp.session.SftpSession;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.messaging.MessageChannel;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author Gary Russell
*/
@ContextConfiguration
@RunWith(SpringJUnit4ClassRunner.class)
@DirtiesContext
public class SftpStreamingInboundChannelAdapterParserTests {
@Autowired
private SourcePollingChannelAdapter sftpInbound;
@Autowired
private MessageChannel sftpChannel;
@Autowired
private CachingSessionFactory<?> csf;
@Test
public void testFtpInboundChannelAdapterComplete() throws Exception {
assertFalse(TestUtils.getPropertyValue(this.sftpInbound, "autoStartup", Boolean.class));
assertEquals("sftpInbound", this.sftpInbound.getComponentName());
assertEquals("sftp:inbound-streaming-channel-adapter", this.sftpInbound.getComponentType());
assertSame(this.sftpChannel, TestUtils.getPropertyValue(this.sftpInbound, "outputChannel"));
SftpStreamingMessageSource source = TestUtils.getPropertyValue(sftpInbound, "source",
SftpStreamingMessageSource.class);
assertNotNull(TestUtils.getPropertyValue(source, "comparator"));
assertThat(TestUtils.getPropertyValue(source, "remoteFileSeparator", String.class), equalTo("X"));
assertThat(TestUtils.getPropertyValue(source, "filter"), instanceOf(SftpSimplePatternFileListFilter.class));
assertSame(this.csf, TestUtils.getPropertyValue(source, "remoteFileTemplate.sessionFactory"));
}
public static class TestSessionFactoryBean implements FactoryBean<DefaultSftpSessionFactory> {
@Override
public DefaultSftpSessionFactory getObject() throws Exception {
DefaultSftpSessionFactory factory = mock(DefaultSftpSessionFactory.class);
SftpSession session = mock(SftpSession.class);
when(factory.getSession()).thenReturn(session);
return factory;
}
@Override
public Class<?> getObjectType() {
return DefaultSftpSessionFactory.class;
}
@Override
public boolean isSingleton() {
return true;
}
}
}

6
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/RollbackLocalFilterTests-context.xml
View File

@@ -7,13 +7,15 @@
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/integration/sftp http://www.springframework.org/schema/integration/sftp/spring-integration-sftp.xsd">
<bean id="extraConfig" class="org.springframework.integration.sftp.inbound.RollbackLocalFilterTests$Config" />
<int-sftp:inbound-channel-adapter id="sftpAdapterAutoCreate"
session-factory="sftpSessionFactory"
channel="requestChannel"
remote-directory-expression="'/sftpSource'"
local-directory="file:local-test-dir/rollback"
auto-create-local-directory="true"
filename-pattern="sftpSource1.txt"
filename-pattern="sftpSource2.txt"
local-filter="acceptOnceFilter">
<int:poller fixed-rate="1000" max-messages-per-poll="2" error-channel="nullChannel">
<int:transactional synchronization-factory="syncFactory" />
@@ -34,6 +36,4 @@
<bean id="transactionManager" class="org.springframework.integration.transaction.PseudoTransactionManager" />
<bean id="sftpServerConfig" class="org.springframework.integration.sftp.TestSftpServerConfig" />
</beans>

22
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/RollbackLocalFilterTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2015 the original author or authors.
* Copyright 2015-2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -30,10 +30,15 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.sftp.SftpTestSupport;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import com.jcraft.jsch.ChannelSftp.LsEntry;
/**
* @author Gary Russell
* @author Artem Bilan
@@ -43,12 +48,12 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@ContextConfiguration
@RunWith(SpringJUnit4ClassRunner.class)
@DirtiesContext
public class RollbackLocalFilterTests {
public class RollbackLocalFilterTests extends SftpTestSupport {
@BeforeClass
@AfterClass
public static void clean() {
new File("local-test-dir/rollback/sftpSource1.txt").delete();
new File("local-test-dir/rollback/sftpSource2.txt").delete();
}
@Autowired
@@ -57,7 +62,7 @@ public class RollbackLocalFilterTests {
@Test
public void testRollback() throws Exception {
assertTrue(this.crash.getLatch().await(10, TimeUnit.SECONDS));
assertEquals("sftpSource1.txt", this.crash.getFile().getName());
assertEquals("sftpSource2.txt", this.crash.getFile().getName());
}
public static class Crash {
@@ -86,4 +91,13 @@ public class RollbackLocalFilterTests {
}
}
public static class Config {
@Bean
public SessionFactory<LsEntry> sftpSessionFactory() {
return RollbackLocalFilterTests.sessionFactory();
}
}
}

128
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSourceTests.java Normal file
View File

@@ -0,0 +1,128 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.sftp.inbound;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import java.io.InputStream;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.integration.annotation.InboundChannelAdapter;
import org.springframework.integration.annotation.Transformer;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.channel.QueueChannel;
import org.springframework.integration.config.EnableIntegration;
import org.springframework.integration.core.MessageSource;
import org.springframework.integration.file.filters.AcceptOnceFileListFilter;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.scheduling.PollerMetadata;
import org.springframework.integration.sftp.SftpTestSupport;
import org.springframework.integration.sftp.session.SftpRemoteFileTemplate;
import org.springframework.integration.transformer.StreamTransformer;
import org.springframework.messaging.Message;
import org.springframework.messaging.PollableChannel;
import org.springframework.messaging.SubscribableChannel;
import org.springframework.scheduling.support.PeriodicTrigger;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import com.jcraft.jsch.ChannelSftp.LsEntry;
/**
* @author Gary Russell
* @since 4.2.12
*
*/
@ContextConfiguration
@RunWith(SpringJUnit4ClassRunner.class)
@DirtiesContext
public class SftpStreamingMessageSourceTests extends SftpTestSupport {
@Autowired
public PollableChannel data;
@SuppressWarnings("unchecked")
@Test
public void testAllContents() {
Message<byte[]> received = (Message<byte[]>) this.data.receive(10000);
assertNotNull(received);
assertThat(new String(received.getPayload()), equalTo("source1"));
received = (Message<byte[]>) this.data.receive(10000);
assertNotNull(received);
assertThat(new String(received.getPayload()), equalTo("source2"));
assertNull(this.data.receive(0));
}
@Configuration
@EnableIntegration
public static class Config {
@Bean
public QueueChannel data() {
return new QueueChannel();
}
@Bean(name = PollerMetadata.DEFAULT_POLLER)
public PollerMetadata defaultPoller() {
PollerMetadata pollerMetadata = new PollerMetadata();
pollerMetadata.setTrigger(new PeriodicTrigger(500));
pollerMetadata.setMaxMessagesPerPoll(2000);
return pollerMetadata;
}
@Bean
@InboundChannelAdapter("stream")
public MessageSource<InputStream> ftpMessageSource() {
SftpStreamingMessageSource messageSource = new SftpStreamingMessageSource(template(), null);
messageSource.setRemoteDirectory("sftpSource/");
messageSource.setFilter(new AcceptOnceFileListFilter<LsEntry>());
return messageSource;
}
@Bean
public SubscribableChannel stream() {
return new DirectChannel();
}
@Bean
@Transformer(inputChannel = "stream", outputChannel = "data")
public org.springframework.integration.transformer.Transformer transformer() {
return new StreamTransformer();
}
@Bean
public SftpRemoteFileTemplate template() {
return new SftpRemoteFileTemplate(ftpSessionFactory());
}
@Bean
public SessionFactory<LsEntry> ftpSessionFactory() {
return SftpStreamingMessageSourceTests.sessionFactory();
}
}
}

29
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/session/SftpRemoteFileTemplateTests.java
View File

@@ -21,19 +21,20 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.expression.common.LiteralExpression;
import org.springframework.integration.file.DefaultFileNameGenerator;
import org.springframework.integration.file.remote.ClientCallbackWithoutResult;
import org.springframework.integration.file.remote.SessionCallback;
import org.springframework.integration.file.remote.SessionCallbackWithoutResult;
import org.springframework.integration.file.remote.session.Session;
import org.springframework.integration.sftp.TestSftpServer;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.sftp.SftpTestSupport;
import org.springframework.integration.sftp.TestSftpServerConfig;
import org.springframework.messaging.support.GenericMessage;
import org.springframework.test.annotation.DirtiesContext;
@@ -53,21 +54,11 @@ import com.jcraft.jsch.SftpException;
@ContextConfiguration(classes=TestSftpServerConfig.class)
@RunWith(SpringJUnit4ClassRunner.class)
@DirtiesContext
public class SftpRemoteFileTemplateTests {
@Autowired
private TestSftpServer sftpServer;
public class SftpRemoteFileTemplateTests extends SftpTestSupport {
@Autowired
private DefaultSftpSessionFactory sessionFactory;
@Before
@After
public void setup() {
this.sftpServer.recursiveDelete(sftpServer.getTargetLocalDirectory());
this.sftpServer.recursiveDelete(sftpServer.getTargetSftpDirectory());
}
@Test
public void testINT3412AppendStatRmdir() {
SftpRemoteFileTemplate template = new SftpRemoteFileTemplate(sessionFactory);
@@ -115,4 +106,14 @@ public class SftpRemoteFileTemplateTests {
assertFalse(template.exists("foo"));
}
@Configuration
public static class Config {
@Bean
public SessionFactory<LsEntry> ftpSessionFactory() {
return SftpRemoteFileTemplateTests.sessionFactory();
}
}
}