Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-2664: (S)FTP Inbound: Preserve File Timestamp

Browse Source 
JIRA: https://jira.springsource.org/browse/INT-2664

INT-2664: add `preserve-timestamp` attribute

Doc Polishing
This commit is contained in:
Artem Bilan
2013-10-28 20:35:03 +02:00
committed by Gary Russell
parent 848f3a5103
commit 498e9aa704
13 changed files with 103 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java
View File

@@ -46,6 +46,7 @@ public abstract class AbstractRemoteFileInboundChannelAdapterParser extends Abst
// configure the InboundFileSynchronizer properties
IntegrationNamespaceUtils.setValueIfAttributeDefined(synchronizerBuilder, element, "remote-directory");
IntegrationNamespaceUtils.setValueIfAttributeDefined(synchronizerBuilder, element, "delete-remote-files");
IntegrationNamespaceUtils.setValueIfAttributeDefined(synchronizerBuilder, element, "preserve-timestamp");
String remoteFileSeparator = element.getAttribute("remote-file-separator");
synchronizerBuilder.addPropertyValue("remoteFileSeparator", remoteFileSeparator);

27
spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java
View File

@@ -19,7 +19,6 @@ package org.springframework.integration.file.remote.synchronizer;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
@@ -50,6 +49,7 @@ import org.springframework.util.ObjectUtils;
* @author Mark Fisher
* @author Oleg Zhurakousky
* @author Gary Russell
* @author Artem Bilan
* @since 2.0
*/
public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileSynchronizer,
@@ -89,6 +89,12 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
*/
private volatile boolean deleteRemoteFiles;
/**
* Should we <em>transfer</em> the remote file <b>timestamp</b>
* to the local file? By default this is false.
*/
private volatile boolean preserveTimestamp;
/**
* Create a synchronizer with the {@link SessionFactory} used to acquire {@link Session} instances.
*/
@@ -127,6 +133,10 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
this.deleteRemoteFiles = deleteRemoteFiles;
}
public void setPreserveTimestamp(boolean preserveTimestamp) {
this.preserveTimestamp = preserveTimestamp;
}
@Override
public void setIntegrationEvaluationContext(EvaluationContext evaluationContext) {
this.evaluationContext = evaluationContext;
@@ -149,7 +159,7 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
Session<F> session = null;
try {
session = this.sessionFactory.getSession();
Assert.state(session != null, "failed to acquire a Session");
Assert.notNull(session, "failed to acquire a Session");
F[] files = session.list(this.remoteDirectory);
if (!ObjectUtils.isEmpty(files)) {
Collection<F> filteredFiles = this.filterFiles(files);
@@ -192,7 +202,6 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
if (!localFile.exists()) {
String tempFileName = localFile.getAbsolutePath() + this.temporaryFileSuffix;
File tempFile = new File(tempFileName);
InputStream inputStream = null;
FileOutputStream fileOutputStream = new FileOutputStream(tempFile);
try {
session.read(remoteFilePath, fileOutputStream);
@@ -206,13 +215,6 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
}
}
finally {
try {
if (inputStream != null) {
inputStream.close();
}
}
catch (Exception ignored1) {
}
try {
fileOutputStream.close();
}
@@ -228,6 +230,9 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
}
}
}
if (this.preserveTimestamp) {
localFile.setLastModified(getModified(remoteFile));
}
}
}
@@ -242,4 +247,6 @@ public abstract class AbstractInboundFileSynchronizer<F> implements InboundFileS
protected abstract String getFilename(F file);
protected abstract long getModified(F file);
}

8
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/inbound/FtpInboundFileSynchronizer.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2010 the original author or authors.
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,6 +28,7 @@ import org.springframework.integration.file.remote.synchronizer.AbstractInboundF
* @author Iwein Fuld
* @author Josh Long
* @author Mark Fisher
* @author Artem Bilan
* @since 2.0
*/
public class FtpInboundFileSynchronizer extends AbstractInboundFileSynchronizer<FTPFile> {
@@ -50,4 +51,9 @@ public class FtpInboundFileSynchronizer extends AbstractInboundFileSynchronizer<
return (file != null ? file.getName() : null);
}
@Override
protected long getModified(FTPFile file) {
return file.getTimestamp().getTimeInMillis();
}
}

12
spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-3.0.xsd
View File

@@ -235,7 +235,7 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="delete-remote-files" type="xsd:string">
<xsd:attribute name="delete-remote-files" type="xsd:string" default="false">
<xsd:annotation>
<xsd:documentation>
Specify whether to delete the remote source
@@ -245,6 +245,16 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="preserve-timestamp" type="xsd:string" default="false">
<xsd:annotation>
<xsd:documentation>
Specify whether to preserve the modified timestamp from the remote source
file on the local file after copying.
By default, the remote timestamp will NOT be
preserved.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>

19
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests-context.xml
View File

@@ -7,20 +7,21 @@
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/integration/ftp http://www.springframework.org/schema/integration/ftp/spring-integration-ftp.xsd">
<bean id="ftpSessionFactory"
<bean id="ftpSessionFactory"
class="org.springframework.integration.ftp.config.FtpInboundChannelAdapterParserTests.TestSessionFactoryBean"/>
<bean id="csf" class="org.springframework.integration.file.remote.session.CachingSessionFactory">
<constructor-arg ref="ftpSessionFactory"/>
</bean>
<int-ftp:inbound-channel-adapter id="ftpInbound"
channel="ftpChannel"
channel="ftpChannel"
session-factory="ftpSessionFactory"
charset="UTF-8"
auto-create-local-directory="true"
auto-startup="false"
delete-remote-files="true"
preserve-timestamp="true"
filename-pattern="*.txt"
local-directory="."
remote-file-separator=""
@@ -39,7 +40,7 @@
</bean>
<bean id="acceptAllFilter" class="org.springframework.integration.file.filters.AcceptAllFileListFilter" />
<int:transaction-synchronization-factory id="syncFactory">
<int:after-commit expression="'foo'" channel="successChannel"/>
<int:after-rollback expression="'bar'" channel="failureChannel"/>
@@ -48,13 +49,13 @@
<int:channel id="successChannel" />
<int:channel id="failureChannel" />
<bean id="comparator" class="org.mockito.Mockito" factory-method="mock">
<constructor-arg value="java.util.Comparator"/>
</bean>
<int-ftp:inbound-channel-adapter
channel="ftpChannel"
channel="ftpChannel"
session-factory="ftpSessionFactory"
charset="UTF-8"
auto-create-local-directory="true"
@@ -66,7 +67,7 @@
</int-ftp:inbound-channel-adapter>
<int-ftp:inbound-channel-adapter id="simpleAdapterWithCachedSessions"
channel="ftpChannel"
channel="ftpChannel"
session-factory="csf"
local-directory="."
remote-directory="foo/bar">
@@ -76,7 +77,7 @@
<int:channel id="ftpChannel">
<int:queue/>
</int:channel>
<bean id="entryListFilter" class="org.mockito.Mockito" factory-method="mock">
<constructor-arg value="org.springframework.integration.file.filters.FileListFilter"/>
</bean>
@@ -89,7 +90,7 @@
</int-ftp:inbound-channel-adapter>
<int:bridge input-channel="autoChannel" output-channel="nullChannel" />
<bean id="transactionManager" class="org.springframework.integration.transaction.PseudoTransactionManager"/>
</beans>

1
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests.java
View File

@@ -77,6 +77,7 @@ public class FtpInboundChannelAdapterParserTests {
FtpInboundFileSynchronizer fisync =
(FtpInboundFileSynchronizer) TestUtils.getPropertyValue(inbound, "synchronizer");
assertNotNull(TestUtils.getPropertyValue(fisync, "localFilenameGeneratorExpression"));
assertTrue(TestUtils.getPropertyValue(fisync, "preserveTimestamp", Boolean.class));
assertEquals(".foo", TestUtils.getPropertyValue(fisync, "temporaryFileSuffix", String.class));
String remoteFileSeparator = (String) TestUtils.getPropertyValue(fisync, "remoteFileSeparator");
assertNotNull(remoteFileSeparator);

14
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpInboundRemoteFileSystemSynchronizerTests.java
View File

@@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
@@ -30,10 +31,12 @@ import static org.mockito.Mockito.when;
import java.io.File;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.Test;
import org.mockito.Mockito;
@@ -51,6 +54,7 @@ import org.springframework.integration.ftp.session.AbstractFtpSessionFactory;
* @author Oleg Zhurakousky
* @author Gunnar Hillert
* @author Gary Russell
* @author Artem Bilan
* @since 2.0
*/
public class FtpInboundRemoteFileSystemSynchronizerTests {
@@ -80,6 +84,7 @@ public class FtpInboundRemoteFileSystemSynchronizerTests {
ftpSessionFactory.setHost("foo.com");
FtpInboundFileSynchronizer synchronizer = spy(new FtpInboundFileSynchronizer(ftpSessionFactory));
synchronizer.setDeleteRemoteFiles(true);
synchronizer.setPreserveTimestamp(true);
synchronizer.setRemoteDirectory("remote-test-dir");
synchronizer.setFilter(new FtpRegexPatternFileListFilter(".*\\.test$"));
synchronizer.setIntegrationEvaluationContext(ExpressionUtils.createStandardEvaluationContext());
@@ -98,9 +103,15 @@ public class FtpInboundRemoteFileSystemSynchronizerTests {
Message<File> atestFile = ms.receive();
assertNotNull(atestFile);
assertEquals("A.TEST.a", atestFile.getPayload().getName());
// The test remote files are created with the current timestamp + 1 day.
assertThat(atestFile.getPayload().lastModified(), Matchers.greaterThan(System.currentTimeMillis()));
Message<File> btestFile = ms.receive();
assertNotNull(btestFile);
assertEquals("B.TEST.a", btestFile.getPayload().getName());
// The test remote files are created with the current timestamp + 1 day.
assertThat(atestFile.getPayload().lastModified(), Matchers.greaterThan(System.currentTimeMillis()));
Message<File> nothing = ms.receive();
assertNull(nothing);
@@ -127,6 +138,9 @@ public class FtpInboundRemoteFileSystemSynchronizerTests {
FTPFile file = new FTPFile();
file.setName(fileName);
file.setType(FTPFile.FILE_TYPE);
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DATE, 1);
file.setTimestamp(calendar);
ftpFiles.add(file);
when(ftpClient.retrieveFile(Mockito.eq("remote-test-dir/" + fileName) , Mockito.any(OutputStream.class))).thenReturn(true);
}

5
spring-integration-sftp/src/main/java/org/springframework/integration/sftp/inbound/SftpInboundFileSynchronizer.java
View File

@@ -46,4 +46,9 @@ public class SftpInboundFileSynchronizer extends AbstractInboundFileSynchronizer
return (file != null ? file.getFilename() : null);
}
@Override
protected long getModified(LsEntry file) {
return (long) file.getAttrs().getMTime() * 1000;
}
}

12
spring-integration-sftp/src/main/resources/org/springframework/integration/sftp/config/spring-integration-sftp-3.0.xsd
View File

@@ -236,7 +236,7 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="delete-remote-files" type="xsd:string">
<xsd:attribute name="delete-remote-files" type="xsd:string" default="false">
<xsd:annotation>
<xsd:documentation>
Specify whether to delete the remote source
@@ -246,6 +246,16 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="preserve-timestamp" type="xsd:string" default="false">
<xsd:annotation>
<xsd:documentation>
Specify whether to preserve the modified timestamp from the remote source
file on the local file after copying.
By default, the remote timestamp will NOT be
preserved.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>

11
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests-context.xml
View File

@@ -22,11 +22,11 @@
<channel id="requestChannel">
<queue/>
</channel>
<beans:bean id="pattern" class="java.util.regex.Pattern" factory-method="compile">
<beans:constructor-arg value="."/>
</beans:bean>
<beans:bean id="sftpSessionFactory" class="org.springframework.integration.sftp.session.DefaultSftpSessionFactory">
<beans:property name="host" value="loclahost"/>
<beans:property name="knownHosts" value="local, foo.com, bar.foo"/>
@@ -49,14 +49,15 @@
temporary-file-suffix=".bar"
comparator="comparator"
local-filter="acceptAllFilter"
delete-remote-files="${delete.remote.files}">
delete-remote-files="${delete.remote.files}"
preserve-timestamp="true">
<poller fixed-rate="1000">
<transactional synchronization-factory="syncFactory"/>
</poller>
</sftp:inbound-channel-adapter>
<beans:bean id="acceptAllFilter" class="org.springframework.integration.file.filters.AcceptAllFileListFilter" />
<transaction-synchronization-factory id="syncFactory">
<after-commit expression="'foo'" channel="successChannel"/>
<after-rollback expression="'bar'" channel="failureChannel"/>
@@ -118,7 +119,7 @@
</sftp:inbound-channel-adapter>
<bridge input-channel="autoChannel" output-channel="nullChannel" />
<beans:bean id="transactionManager" class="org.springframework.integration.transaction.PseudoTransactionManager"/>
</beans:beans>

1
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests.java
View File

@@ -82,6 +82,7 @@ public class InboundChannelAdapterParserTests {
assertNotNull(comparator);
SftpInboundFileSynchronizer synchronizer = (SftpInboundFileSynchronizer) TestUtils.getPropertyValue(source, "synchronizer");
assertNotNull(TestUtils.getPropertyValue(synchronizer, "localFilenameGeneratorExpression"));
assertTrue(TestUtils.getPropertyValue(synchronizer, "preserveTimestamp", Boolean.class));
String remoteFileSeparator = (String) TestUtils.getPropertyValue(synchronizer, "remoteFileSeparator");
assertEquals(".bar", TestUtils.getPropertyValue(synchronizer, "temporaryFileSuffix", String.class));
assertNotNull(remoteFileSeparator);

15
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpInboundRemoteFileSystemSynchronizerTests.java
View File

@@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
@@ -29,8 +30,10 @@ import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileInputStream;
import java.util.Calendar;
import java.util.Vector;
import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.Test;
@@ -49,6 +52,7 @@ import com.jcraft.jsch.SftpATTRS;
* @author Oleg Zhurakousky
* @author Gunnar Hillert
* @author Gary Russell
* @author Artem Bilan
* @since 2.0
*/
public class SftpInboundRemoteFileSystemSynchronizerTests {
@@ -81,6 +85,7 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
SftpInboundFileSynchronizer synchronizer = spy(new SftpInboundFileSynchronizer(ftpSessionFactory));
synchronizer.setDeleteRemoteFiles(true);
synchronizer.setPreserveTimestamp(true);
synchronizer.setRemoteDirectory("remote-test-dir");
synchronizer.setFilter(new SftpRegexPatternFileListFilter(".*\\.test$"));
synchronizer.setIntegrationEvaluationContext(ExpressionUtils.createStandardEvaluationContext());
@@ -93,9 +98,15 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
Message<File> atestFile = ms.receive();
assertNotNull(atestFile);
assertEquals("a.test", atestFile.getPayload().getName());
// The test remote files are created with the current timestamp + 1 day.
assertThat(atestFile.getPayload().lastModified(), Matchers.greaterThan(System.currentTimeMillis()));
Message<File> btestFile = ms.receive();
assertNotNull(btestFile);
assertEquals("b.test", btestFile.getPayload().getName());
// The test remote files are created with the current timestamp + 1 day.
assertThat(atestFile.getPayload().lastModified(), Matchers.greaterThan(System.currentTimeMillis()));
Message<File> nothing = ms.receive();
assertNull(nothing);
@@ -120,6 +131,10 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
LsEntry lsEntry = mock(LsEntry.class);
SftpATTRS attributes = mock(SftpATTRS.class);
when(lsEntry.getAttrs()).thenReturn(attributes);
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DATE, 1);
when(lsEntry.getAttrs().getMTime()).thenReturn(new Long(calendar.getTimeInMillis() / 1000).intValue());
when(lsEntry.getFilename()).thenReturn(fileName);
sftpEntries.add(lsEntry);
when(channel.get("remote-test-dir/"+fileName)).thenReturn(new FileInputStream("remote-test-dir/" + fileName));

4
src/reference/docbook/whats-new.xml
View File

@@ -248,6 +248,10 @@
to be maintained across JVM executions, a custom filter that retains state, perhaps on
the file system, can now be configured.
</para>
<para>
Inbound Channel Adapters now support the <code>preserve-timestamp</code> attribute, which
sets the local file modified timestamp to the timestamp from the server (default false).
</para>
<para>
For more information, see
<xref linkend="ftp-inbound"/> and <xref linkend="sftp-inbound"/>.