Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #139 from olegz/INT-1844

Browse Source 
Added support for generating local file name for FTP and SFTP inbound adapters
This commit is contained in:
Mark Fisher
2011-10-19 15:48:55 -04:00
parent 6af7d5cd91 b28ec17885
commit 761a797ee1
11 changed files with 121 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
docs/src/reference/docbook/ftp.xml
View File

@@ -116,6 +116,7 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/ftp
filename-pattern="*.txt"
remote-directory="some/remote/path"
remote-file-separator="/"
local-filename-generator-expression="#this.toUpperCase() + '.a'"
local-directory=".">
<int:poller fixed-rate="1000"/>
</int-ftp:inbound-channel-adapter>]]></programlisting>
@@ -124,6 +125,15 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/ftp
element while also providing values for various attributes such as <code>local-directory</code>, <code>filename-pattern</code>
(which is based on simple pattern matching, not regular expressions), and of course the reference to a <code>session-factory</code>.
</para>
<para>
By default the transferred file will carry the same name as the original file. If you want to override this behavior you
can set the <code>local-filename-generator-expression</code> attribute which allows you to provide a SpEL Expression to generate
the name of the local file. Unlike outbound gateways and adapters where the root object of the SpEL Evaluation Context
is a <classname>Message</classname>, this inbound adapter does not yet have the Message at the time of evaluation since
that's what it ultimately generates with the transferred file as its payload. So, the root object of the SpEL Evaluation Context
is the original name of the remote file (String).
</para>
<para>
Some times file filtering based on the simple pattern specified via <code>filename-pattern</code> attribute might not be
sufficient. If this is the case, you can use the <code>filename-regex</code> attribute to specify a Regular Expression

9
docs/src/reference/docbook/sftp.xml
View File

@@ -80,6 +80,7 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp
remote-directory="/foo/bar"
local-directory="file:target/foo"
auto-create-local-directory="true"
local-filename-generator-expression="#this.toUpperCase() + '.a'"
delete-remote-files="false">
<int:poller fixed-rate="1000"/>
</int-sftp:inbound-channel-adapter>]]></programlisting>
@@ -90,6 +91,14 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp
going to be transferred FROM -
as well as other attributes including a <code>session-factory</code> reference to the bean we configured earlier.
</para>
<para>
By default the transferred file will carry the same name as the original file. If you want to override this behavior you
can set the <code>local-filename-generator-expression</code> attribute which allows you to provide a SpEL Expression to generate
the name of the local file. Unlike outbound gateways and adapters where the root object of the SpEL Evaluation Context
is a <classname>Message</classname>, this inbound adapter does not yet have the Message at the time of evaluation since
that's what it ultimately generates with the transferred file as its payload. So, the root object of the SpEL Evaluation Context
is the original name of the remote file (String).
</para>
<para>
Some times file filtering based on the simple pattern specified via <code>filename-pattern</code> attribute might not be
sufficient. If this is the case, you can use the <code>filename-regex</code> attribute to specify a Regular Expression

9
spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java
View File

@@ -21,6 +21,7 @@ import org.w3c.dom.Element;
import org.springframework.beans.BeanMetadataElement;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.ExpressionFactoryBean;
import org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.util.StringUtils;
@@ -54,6 +55,14 @@ public abstract class AbstractRemoteFileInboundChannelAdapterParser extends Abst
// configure the InboundFileSynchronizer properties
IntegrationNamespaceUtils.setValueIfAttributeDefined(synchronizerBuilder, element, "remote-directory");
IntegrationNamespaceUtils.setValueIfAttributeDefined(synchronizerBuilder, element, "delete-remote-files");
String localFileGeneratorExpression = element.getAttribute("local-filename-generator-expression");
if (StringUtils.hasText(localFileGeneratorExpression)){
BeanDefinitionBuilder localFileGeneratorExpressionBuilder = BeanDefinitionBuilder.genericBeanDefinition(ExpressionFactoryBean.class);
localFileGeneratorExpressionBuilder.addConstructorArgValue(localFileGeneratorExpression);
synchronizerBuilder.addPropertyValue("localFilenameGeneratorExpression", localFileGeneratorExpressionBuilder.getBeanDefinition());
}
String remoteFileSeparator = element.getAttribute("remote-file-separator");
synchronizerBuilder.addPropertyValue("remoteFileSeparator", remoteFileSeparator);
IntegrationNamespaceUtils.setValueIfAttributeDefined(synchronizerBuilder, element, "temporary-file-suffix");

32
spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java
View File

@@ -28,6 +28,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.expression.Expression;
import org.springframework.expression.spel.support.StandardEvaluationContext;
import org.springframework.integration.MessagingException;
import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.file.remote.session.Session;
@@ -50,13 +52,18 @@ import org.springframework.util.ObjectUtils;
*/
public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileSynchronizer, InitializingBean {
private String remoteFileSeparator = "/";
protected final Log logger = LogFactory.getLog(this.getClass());
private final StandardEvaluationContext evaluationContext = new StandardEvaluationContext();
private volatile String remoteFileSeparator = "/";
/**
* Extension used when downloading files. We change it right after we know it's downloaded.
*/
private volatile String temporaryFileSuffix =".writing";
protected final Log logger = LogFactory.getLog(this.getClass());
private volatile Expression localFilenameGeneratorExpression;
/**
* the path on the remote mount as a String.
@@ -77,7 +84,7 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
* Should we <emphasis>delete</emphasis> the remote <b>source</b> files
* after copying to the local directory? By default this is false.
*/
private boolean deleteRemoteFiles;
private volatile boolean deleteRemoteFiles;
/**
@@ -93,7 +100,12 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
Assert.notNull(remoteFileSeparator, "'remoteFileSeparator' must not be null");
this.remoteFileSeparator = remoteFileSeparator;
}
public void setLocalFilenameGeneratorExpression(Expression localFilenameGeneratorExpression) {
Assert.notNull(localFilenameGeneratorExpression, "'localFilenameGeneratorExpression' must not be null");
this.localFilenameGeneratorExpression = localFilenameGeneratorExpression;
}
public void setTemporaryFileSuffix(String temporaryFileSuffix) {
this.temporaryFileSuffix = temporaryFileSuffix;
}
@@ -159,6 +171,7 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
private void copyFileToLocalDirectory(String remoteDirectoryPath, F remoteFile, File localDirectory, Session session) throws IOException {
String remoteFileName = this.getFilename(remoteFile);
String localFileName = this.generateLocalFileName(remoteFileName);
String remoteFilePath = remoteDirectoryPath + remoteFileSeparator + remoteFileName;
if (!this.isFile(remoteFile)) {
if (logger.isDebugEnabled()) {
@@ -166,7 +179,8 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
}
return;
}
File localFile = new File(localDirectory, remoteFileName);
File localFile = new File(localDirectory, localFileName);
if (!localFile.exists()) {
String tempFileName = localFile.getAbsolutePath() + this.temporaryFileSuffix;
File tempFile = new File(tempFileName);
@@ -197,6 +211,7 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
catch (Exception ignored2) {
}
}
if (tempFile.renameTo(localFile)) {
if (this.deleteRemoteFiles) {
session.remove(remoteFilePath);
@@ -207,6 +222,13 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
}
}
}
private String generateLocalFileName(String remoteFileName){
if (this.localFilenameGeneratorExpression != null){
return this.localFilenameGeneratorExpression.getValue(evaluationContext, remoteFileName, String.class);
}
return remoteFileName;
}
protected abstract boolean isFile(F file);

26
spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-2.1.xsd
View File

@@ -110,15 +110,25 @@ endpoint itself is a Polling Consumer for a channel with a queue.
<xsd:attribute name="filename-pattern" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide file name pattern to determine the file names that needs to be scanned
and is based on simple pattern matching algorithm (e.g., "*.txt, fo*.txt" etc.)
Allows you to provide a file name pattern to determine the file names that need to be scanned.
This is based on simple pattern matching (e.g., "*.txt, fo*.txt" etc.)
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="local-filename-generator-expression" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide a SpEL expression to generate the file name of
the local (transferred) file. The root object of the SpEL evaluation is the name of the original
file. For example, a valid expression would be "#this.toUpperCase() + '.a'" where #this represents the
original name of the remote file.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="filename-regex" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide Regular Expression to determine the file names that needs to be scanned.
Allows you to provide a Regular Expression to determine the file names that need to be scanned.
(e.g., "f[o]+\.txt" etc.)
</xsd:documentation>
</xsd:annotation>
@@ -147,7 +157,7 @@ endpoint itself is a Polling Consumer for a channel with a queue.
</tool:annotation>
</xsd:appinfo>
<xsd:documentation>
Allows you to specify a reference to
Allows you to specify a reference to a
[org.springframework.integration.file.filters.FileListFilter] bean.
</xsd:documentation>
</xsd:annotation>
@@ -162,28 +172,28 @@ endpoint itself is a Polling Consumer for a channel with a queue.
<xsd:attribute name="remote-directory" type="xsd:string" use="required">
<xsd:annotation>
<xsd:documentation>
Identifies directory path (e.g., "/temp/mytransfers") where file will be transferred FROM.
Identifies the directory path (e.g., "/temp/mytransfers") where files will be transferred FROM.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="remote-file-separator" type="xsd:string" default="/">
<xsd:annotation>
<xsd:documentation>
Allows you to provide remote file/directory separator character. DEFAULT: '/'
Allows you to provide a remote file/directory separator character. DEFAULT: '/'
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="local-directory" type="xsd:string" use="required">
<xsd:annotation>
<xsd:documentation>
Identifies directory path (e.g., "/local/mytransfers") where file will be transferred TO.
Identifies the directory path (e.g., "/local/mytransfers") where files will be transferred TO.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="auto-create-local-directory" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Tells this adapter if local directory must be auto-created if it doesn''t exist. Default is TRUE.
Tells this adapter if the local directory must be auto-created if it doesn't exist. Default is TRUE.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>

1
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests-context.xml
View File

@@ -21,6 +21,7 @@
filename-pattern="*.txt"
local-directory="."
remote-file-separator=""
local-filename-generator-expression="#this.toUpperCase() + '.a'"
comparator="comparator"
temporary-file-suffix=".foo"
remote-directory="foo/bar">

1
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests.java
View File

@@ -66,6 +66,7 @@ public class FtpInboundChannelAdapterParserTests {
FtpInboundFileSynchronizer fisync =
(FtpInboundFileSynchronizer) TestUtils.getPropertyValue(inbound, "synchronizer");
assertNotNull(TestUtils.getPropertyValue(fisync, "localFilenameGeneratorExpression"));
assertEquals(".foo", TestUtils.getPropertyValue(fisync, "temporaryFileSuffix", String.class));
String remoteFileSeparator = (String) TestUtils.getPropertyValue(fisync, "remoteFileSeparator");
assertNotNull(remoteFileSeparator);

40
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpInboundRemoteFileSystemSynchronizerTest.java
View File

@@ -16,17 +16,6 @@
package org.springframework.integration.ftp.inbound;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertNotNull;
import static junit.framework.Assert.assertNull;
import static junit.framework.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.OutputStream;
import java.util.ArrayList;
@@ -37,11 +26,25 @@ import org.apache.commons.net.ftp.FTPFile;
import org.junit.After;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.expression.Expression;
import org.springframework.expression.ExpressionParser;
import org.springframework.expression.spel.SpelParserConfiguration;
import org.springframework.expression.spel.standard.SpelExpressionParser;
import org.springframework.integration.Message;
import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter;
import org.springframework.integration.ftp.session.AbstractFtpSessionFactory;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertNotNull;
import static junit.framework.Assert.assertNull;
import static junit.framework.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* @author Oleg Zhurakousky
* @since 2.0
@@ -76,27 +79,32 @@ public class FtpInboundRemoteFileSystemSynchronizerTest {
synchronizer.setDeleteRemoteFiles(true);
synchronizer.setRemoteDirectory("remote-test-dir");
synchronizer.setFilter(new FtpRegexPatternFileListFilter(".*\\.test$"));
ExpressionParser expressionParser = new SpelExpressionParser(new SpelParserConfiguration(true, true));
Expression expression = expressionParser.parseExpression("#this.toUpperCase() + '.a'");
synchronizer.setLocalFilenameGeneratorExpression(expression);
FtpInboundFileSynchronizingMessageSource ms =
new FtpInboundFileSynchronizingMessageSource(synchronizer);
ms.setAutoCreateLocalDirectory(true);
ms.setLocalDirectory(localDirectoy);
ms.afterPropertiesSet();
Message<File> atestFile = ms.receive();
assertNotNull(atestFile);
assertEquals("a.test", atestFile.getPayload().getName());
assertEquals("A.TEST.a", atestFile.getPayload().getName());
Message<File> btestFile = ms.receive();
assertNotNull(btestFile);
assertEquals("b.test", btestFile.getPayload().getName());
assertEquals("B.TEST.a", btestFile.getPayload().getName());
Message<File> nothing = ms.receive();
assertNull(nothing);
// two times because on the third receive (above) the internal queue will be empty, so it will attempt
verify(synchronizer, times(2)).synchronizeToLocalDirectory(localDirectoy);
assertTrue(new File("test/a.test").exists());
assertTrue(new File("test/b.test").exists());
assertTrue(new File("test/A.TEST.a").exists());
assertTrue(new File("test/B.TEST.a").exists());
}

30
spring-integration-sftp/src/main/resources/org/springframework/integration/sftp/config/spring-integration-sftp-2.1.xsd
View File

@@ -195,8 +195,8 @@ endpoint itself is a Polling Consumer for a channel with a queue.
</tool:annotation>
</xsd:appinfo>
<xsd:documentation>
Identifies channel attached to this adapter. Depending on the type of the adapter
this channel could be the receiving channel (e.g., outbound-channel-adapter) or channel where
Identifies the channel attached to this adapter. Depending on the type of the adapter
this channel could be the receiving channel (e.g., outbound-channel-adapter) or the channel where
messages will be sent to by this adapter (e.g., inbound-channel-adapter).
</xsd:documentation>
</xsd:annotation>
@@ -209,7 +209,7 @@ endpoint itself is a Polling Consumer for a channel with a queue.
</tool:annotation>
</xsd:appinfo>
<xsd:documentation>
Allows you to specify a reference to
Allows you to specify a reference to a
[org.springframework.integration.file.filters.FileListFilter] bean.
</xsd:documentation>
</xsd:annotation>
@@ -217,15 +217,25 @@ endpoint itself is a Polling Consumer for a channel with a queue.
<xsd:attribute name="filename-pattern" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide file name pattern to determine the file names that needs to be scanned
and is based on simple pattern matching algorithm (e.g., "*.txt, fo*.txt" etc.)
Allows you to provide a file name pattern to determine the file names that need to be scanned.
This is based on simple pattern matching (e.g., "*.txt, fo*.txt" etc.)
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="local-filename-generator-expression" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide a SpEL expression to generate the file name of
the local (transferred) file. The root object of the SpEL evaluation is the name of the original
file. For example, a valid expression would be "#this.toUpperCase() + '.a'" where #this represents the
original name of the remote file.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="filename-regex" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide Regular Expression to determine the file names that needs to be scanned.
Allows you to provide a Regular Expression to determine the file names that need to be scanned.
(e.g., "f[o]+\.txt" etc.)
</xsd:documentation>
</xsd:annotation>
@@ -233,28 +243,28 @@ endpoint itself is a Polling Consumer for a channel with a queue.
<xsd:attribute name="remote-file-separator" type="xsd:string" default="/">
<xsd:annotation>
<xsd:documentation>
Allows you to provide remote file/directory separator character. DEFAULT: '/'
Allows you to provide a remote file/directory separator character. DEFAULT: '/'
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="remote-directory" type="xsd:string" use="required">
<xsd:annotation>
<xsd:documentation>
Identifies directory path (e.g., "/temp/mytransfers") where file will be transferred FROM.
Identifies the directory path (e.g., "/temp/mytransfers") where files will be transferred FROM.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="local-directory" type="xsd:string" use="required">
<xsd:annotation>
<xsd:documentation>
Identifies directory path (e.g., "/local/mytransfers") where file will be transferred TO.
Identifies the directory path (e.g., "/local/mytransfers") where files will be transferred TO.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="auto-create-local-directory" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Tells this adapter if local directory must be auto-created if it doesn''t exist. Default is TRUE.
Tells this adapter if the local directory must be auto-created if it doesn't exist. Default is TRUE.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>

1
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests-context.xml
View File

@@ -45,6 +45,7 @@
local-directory="file:local-test-dir"
auto-create-local-directory="false"
remote-file-separator="."
local-filename-generator-expression="#this.toUpperCase() + '.a'"
temporary-file-suffix=".bar"
comparator="comparator"
delete-remote-files="${delete.remote.files}">

1
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests.java
View File

@@ -73,6 +73,7 @@ public class InboundChannelAdapterParserTests {
Comparator<File> comparator = TestUtils.getPropertyValue(adapter, "source.fileSource.toBeReceived.q.comparator", Comparator.class);
assertNotNull(comparator);
SftpInboundFileSynchronizer synchronizer = (SftpInboundFileSynchronizer) TestUtils.getPropertyValue(source, "synchronizer");
assertNotNull(TestUtils.getPropertyValue(synchronizer, "localFilenameGeneratorExpression"));
String remoteFileSeparator = (String) TestUtils.getPropertyValue(synchronizer, "remoteFileSeparator");
assertEquals(".bar", TestUtils.getPropertyValue(synchronizer, "temporaryFileSuffix", String.class));
assertNotNull(remoteFileSeparator);