Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-1844 added support for file name generation

Browse Source 
Added support for generating local file name for FTP and SFTP inbound adapters
This commit is contained in:
Oleg Zhurakousky
2011-10-18 17:33:20 -04:00
committed by Mark Fisher
parent 6af7d5cd91
commit 5059ecc318
11 changed files with 96 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
docs/src/reference/docbook/ftp.xml
View File

@@ -116,6 +116,7 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/ftp
filename-pattern="*.txt"
remote-directory="some/remote/path"
remote-file-separator="/"
local-filename-generator-expression="#this.toUpperCase() + '.a'"
local-directory=".">
<int:poller fixed-rate="1000"/>
</int-ftp:inbound-channel-adapter>]]></programlisting>
@@ -124,6 +125,15 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/ftp
element while also providing values for various attributes such as <code>local-directory</code>, <code>filename-pattern</code>
(which is based on simple pattern matching, not regular expressions), and of course the reference to a <code>session-factory</code>.
</para>
<para>
By default the transferred file will carry the same name as the original file. If you want to override this behavior you
can set <code>local-filename-generator-expression</code> attribute which allows you to provide SpEL Expression to generate
the name of the local file. Unlike outbound gateways and adapters where the root object of the SpEL Evaluation Context
is <classname>Message</classname>, Inbound adapter generates <classname>Message</classname> based on the transferred file
which means that <classname>Message</classname> does not exist yet at the time of Expression evaluation. So the root object
of the SpEL Evaluation Context is the original name of the remote file (String)
</para>
<para>
Some times file filtering based on the simple pattern specified via <code>filename-pattern</code> attribute might not be
sufficient. If this is the case, you can use the <code>filename-regex</code> attribute to specify a Regular Expression

9
docs/src/reference/docbook/sftp.xml
View File

@@ -80,6 +80,7 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp
remote-directory="/foo/bar"
local-directory="file:target/foo"
auto-create-local-directory="true"
local-filename-generator-expression="#this.toUpperCase() + '.a'"
delete-remote-files="false">
<int:poller fixed-rate="1000"/>
</int-sftp:inbound-channel-adapter>]]></programlisting>
@@ -90,6 +91,14 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp
going to be transferred FROM -
as well as other attributes including a <code>session-factory</code> reference to the bean we configured earlier.
</para>
<para>
By default the transferred file will carry the same name as the original file. If you want to override this behavior you
can set <code>local-filename-generator-expression</code> attribute which allows you to provide SpEL Expression to generate
the name of the local file. Unlike outbound gateways and adapters where the root object of the SpEL Evaluation Context
is <classname>Message</classname>, Inbound adapter generates <classname>Message</classname> based on the transferred file
which means that <classname>Message</classname> does not exist yet at the time of Expression evaluation. So the root object
of the SpEL Evaluation Context is the original name of the remote file (String)
</para>
<para>
Some times file filtering based on the simple pattern specified via <code>filename-pattern</code> attribute might not be
sufficient. If this is the case, you can use the <code>filename-regex</code> attribute to specify a Regular Expression

9
spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java
View File

@@ -21,6 +21,7 @@ import org.w3c.dom.Element;
import org.springframework.beans.BeanMetadataElement;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.ExpressionFactoryBean;
import org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.util.StringUtils;
@@ -54,6 +55,14 @@ public abstract class AbstractRemoteFileInboundChannelAdapterParser extends Abst
// configure the InboundFileSynchronizer properties
IntegrationNamespaceUtils.setValueIfAttributeDefined(synchronizerBuilder, element, "remote-directory");
IntegrationNamespaceUtils.setValueIfAttributeDefined(synchronizerBuilder, element, "delete-remote-files");
String localFileGeneratorExpression = element.getAttribute("local-filename-generator-expression");
if (StringUtils.hasText(localFileGeneratorExpression)){
BeanDefinitionBuilder localFileGeneratorExpressionBuilder = BeanDefinitionBuilder.genericBeanDefinition(ExpressionFactoryBean.class);
localFileGeneratorExpressionBuilder.addConstructorArgValue(localFileGeneratorExpression);
synchronizerBuilder.addPropertyValue("localFilenameGeneratorExpression", localFileGeneratorExpressionBuilder.getBeanDefinition());
}
String remoteFileSeparator = element.getAttribute("remote-file-separator");
synchronizerBuilder.addPropertyValue("remoteFileSeparator", remoteFileSeparator);
IntegrationNamespaceUtils.setValueIfAttributeDefined(synchronizerBuilder, element, "temporary-file-suffix");

24
spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java
View File

@@ -26,8 +26,9 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.expression.Expression;
import org.springframework.expression.spel.support.StandardEvaluationContext;
import org.springframework.integration.MessagingException;
import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.file.remote.session.Session;
@@ -49,6 +50,8 @@ import org.springframework.util.ObjectUtils;
* @since 2.0
*/
public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileSynchronizer, InitializingBean {
private static final StandardEvaluationContext context = new StandardEvaluationContext();
private String remoteFileSeparator = "/";
/**
@@ -57,6 +60,8 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
private volatile String temporaryFileSuffix =".writing";
protected final Log logger = LogFactory.getLog(this.getClass());
private volatile Expression localFilenameGeneratorExpression;
/**
* the path on the remote mount as a String.
@@ -80,6 +85,11 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
private boolean deleteRemoteFiles;
public void setLocalFilenameGeneratorExpression(Expression localFilenameGeneratorExpression) {
Assert.notNull(localFilenameGeneratorExpression, "'localFilenameGeneratorExpression' must not be null");
this.localFilenameGeneratorExpression = localFilenameGeneratorExpression;
}
/**
* Create a synchronizer with the {@link SessionFactory} used to acquire {@link Session} instances.
*/
@@ -159,6 +169,7 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
private void copyFileToLocalDirectory(String remoteDirectoryPath, F remoteFile, File localDirectory, Session session) throws IOException {
String remoteFileName = this.getFilename(remoteFile);
String localFileName = this.generateLocalFileName(remoteFileName);
String remoteFilePath = remoteDirectoryPath + remoteFileSeparator + remoteFileName;
if (!this.isFile(remoteFile)) {
if (logger.isDebugEnabled()) {
@@ -166,7 +177,8 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
}
return;
}
File localFile = new File(localDirectory, remoteFileName);
File localFile = new File(localDirectory, localFileName);
if (!localFile.exists()) {
String tempFileName = localFile.getAbsolutePath() + this.temporaryFileSuffix;
File tempFile = new File(tempFileName);
@@ -197,6 +209,7 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
catch (Exception ignored2) {
}
}
if (tempFile.renameTo(localFile)) {
if (this.deleteRemoteFiles) {
session.remove(remoteFilePath);
@@ -207,6 +220,13 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
}
}
}
private String generateLocalFileName(String remoteFileName){
if (this.localFilenameGeneratorExpression != null){
return this.localFilenameGeneratorExpression.getValue(context, remoteFileName, String.class);
}
return remoteFileName;
}
protected abstract boolean isFile(F file);

9
spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-2.1.xsd
View File

@@ -115,6 +115,15 @@ endpoint itself is a Polling Consumer for a channel with a queue.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="local-filename-generator-expression" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide SpEL expression which will compute the file name of
the local file (transfered file). Root object of SpEL is the name of the original
file (e.g., "#this.toUpperCase() + '.a'" where #this represents the original name of the remote file)
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="filename-regex" type="xsd:string">
<xsd:annotation>
<xsd:documentation>

1
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests-context.xml
View File

@@ -21,6 +21,7 @@
filename-pattern="*.txt"
local-directory="."
remote-file-separator=""
local-filename-generator-expression="#this.toUpperCase() + '.a'"
comparator="comparator"
temporary-file-suffix=".foo"
remote-directory="foo/bar">

1
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests.java
View File

@@ -66,6 +66,7 @@ public class FtpInboundChannelAdapterParserTests {
FtpInboundFileSynchronizer fisync =
(FtpInboundFileSynchronizer) TestUtils.getPropertyValue(inbound, "synchronizer");
assertNotNull(TestUtils.getPropertyValue(fisync, "localFilenameGeneratorExpression"));
assertEquals(".foo", TestUtils.getPropertyValue(fisync, "temporaryFileSuffix", String.class));
String remoteFileSeparator = (String) TestUtils.getPropertyValue(fisync, "remoteFileSeparator");
assertNotNull(remoteFileSeparator);

40
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpInboundRemoteFileSystemSynchronizerTest.java
View File

@@ -16,17 +16,6 @@
package org.springframework.integration.ftp.inbound;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertNotNull;
import static junit.framework.Assert.assertNull;
import static junit.framework.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.OutputStream;
import java.util.ArrayList;
@@ -37,11 +26,25 @@ import org.apache.commons.net.ftp.FTPFile;
import org.junit.After;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.expression.Expression;
import org.springframework.expression.ExpressionParser;
import org.springframework.expression.spel.SpelParserConfiguration;
import org.springframework.expression.spel.standard.SpelExpressionParser;
import org.springframework.integration.Message;
import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter;
import org.springframework.integration.ftp.session.AbstractFtpSessionFactory;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertNotNull;
import static junit.framework.Assert.assertNull;
import static junit.framework.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* @author Oleg Zhurakousky
* @since 2.0
@@ -76,27 +79,32 @@ public class FtpInboundRemoteFileSystemSynchronizerTest {
synchronizer.setDeleteRemoteFiles(true);
synchronizer.setRemoteDirectory("remote-test-dir");
synchronizer.setFilter(new FtpRegexPatternFileListFilter(".*\\.test$"));
ExpressionParser expressionParser = new SpelExpressionParser(new SpelParserConfiguration(true, true));
Expression expression = expressionParser.parseExpression("#this.toUpperCase() + '.a'");
synchronizer.setLocalFilenameGeneratorExpression(expression);
FtpInboundFileSynchronizingMessageSource ms =
new FtpInboundFileSynchronizingMessageSource(synchronizer);
ms.setAutoCreateLocalDirectory(true);
ms.setLocalDirectory(localDirectoy);
ms.afterPropertiesSet();
Message<File> atestFile = ms.receive();
assertNotNull(atestFile);
assertEquals("a.test", atestFile.getPayload().getName());
assertEquals("A.TEST.a", atestFile.getPayload().getName());
Message<File> btestFile = ms.receive();
assertNotNull(btestFile);
assertEquals("b.test", btestFile.getPayload().getName());
assertEquals("B.TEST.a", btestFile.getPayload().getName());
Message<File> nothing = ms.receive();
assertNull(nothing);
// two times because on the third receive (above) the internal queue will be empty, so it will attempt
verify(synchronizer, times(2)).synchronizeToLocalDirectory(localDirectoy);
assertTrue(new File("test/a.test").exists());
assertTrue(new File("test/b.test").exists());
assertTrue(new File("test/A.TEST.a").exists());
assertTrue(new File("test/B.TEST.a").exists());
}

9
spring-integration-sftp/src/main/resources/org/springframework/integration/sftp/config/spring-integration-sftp-2.1.xsd
View File

@@ -222,6 +222,15 @@ endpoint itself is a Polling Consumer for a channel with a queue.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="local-filename-generator-expression" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Allows you to provide SpEL expression which will compute the file name of
the local file (transfered file). Root object of SpEL is the name of the original
file (e.g., "#this.toUpperCase() + '.a'" where #this represents the original name of the remote file)
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="filename-regex" type="xsd:string">
<xsd:annotation>
<xsd:documentation>

1
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests-context.xml
View File

@@ -45,6 +45,7 @@
local-directory="file:local-test-dir"
auto-create-local-directory="false"
remote-file-separator="."
local-filename-generator-expression="#this.toUpperCase() + '.a'"
temporary-file-suffix=".bar"
comparator="comparator"
delete-remote-files="${delete.remote.files}">

1
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests.java
View File

@@ -73,6 +73,7 @@ public class InboundChannelAdapterParserTests {
Comparator<File> comparator = TestUtils.getPropertyValue(adapter, "source.fileSource.toBeReceived.q.comparator", Comparator.class);
assertNotNull(comparator);
SftpInboundFileSynchronizer synchronizer = (SftpInboundFileSynchronizer) TestUtils.getPropertyValue(source, "synchronizer");
assertNotNull(TestUtils.getPropertyValue(synchronizer, "localFilenameGeneratorExpression"));
String remoteFileSeparator = (String) TestUtils.getPropertyValue(synchronizer, "remoteFileSeparator");
assertEquals(".bar", TestUtils.getPropertyValue(synchronizer, "temporaryFileSuffix", String.class));
assertNotNull(remoteFileSeparator);