Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-4304 (S)FTP Outbound Gateway Delete After GET

Browse Source 
JIRA: https://jira.spring.io/browse/INT-4304

Add an option to delete remote file(s) after GET/MGET.

Polishing - log error on delete failure.
This commit is contained in:
Gary Russell
2017-06-23 15:38:03 -04:00
committed by Artem Bilan
parent a1c42f65d0
commit 5216dc2b02
6 changed files with 80 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java
View File

@@ -79,42 +79,42 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
public enum Command {
/**
* List remote files.
* (ls) List remote files.
*/
LS("ls"),
/**
* List remote file names.
* (nlst) List remote file names.
*/
NLST("nlst"),
/**
* Retrieve a remote file.
* (get) Retrieve a remote file.
*/
GET("get"),
/**
* Remove a remote file (path - including wildcards).
* (rm) Remove a remote file (path - including wildcards).
*/
RM("rm"),
/**
* Retrieve multiple files matching a wildcard path.
* (mget) Retrieve multiple files matching a wildcard path.
*/
MGET("mget"),
/**
* Move (rename) a remote file.
* (mv) Move (rename) a remote file.
*/
MV("mv"),
/**
* Put a local file to the remote system.
* (put) Put a local file to the remote system.
*/
PUT("put"),
/**
* Put multiple local files to the remote system.
* (mput) Put multiple local files to the remote system.
*/
MPUT("mput");
@@ -146,49 +146,55 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
public enum Option {
/**
* Don't return full file information; just the name (ls).
* (-1) Don't return full file information; just the name (ls).
*/
NAME_ONLY("-1"),
/**
* Include files beginning with {@code .}, including directories {@code .} and {@code ..} in the results (ls).
* (-a) Include files beginning with {@code .}, including directories {@code .}
* and {@code ..} in the results (ls).
*/
ALL("-a"),
/**
* Do not sort the results (ls with NAME_ONLY).
* (-f) Do not sort the results (ls with NAME_ONLY).
*/
NOSORT("-f"),
/**
* Include directories in the results (ls).
* (-dirs) Include directories in the results (ls).
*/
SUBDIRS("-dirs"),
/**
* Include links in the results (ls).
* (-links) Include links in the results (ls).
*/
LINKS("-links"),
/**
* Preserve the server timestamp (get, mget).
* (-P) Preserve the server timestamp (get, mget).
*/
PRESERVE_TIMESTAMP("-P"),
/**
* Throw an exception if no files returned (mget).
* (-x) Throw an exception if no files returned (mget).
*/
EXCEPTION_WHEN_EMPTY("-x"),
/**
* Recursive (ls, mget)
* (-R) Recursive (ls, mget)
*/
RECURSIVE("-R"),
/**
* Streaming 'get' (returns InputStream); user must call {@link Session#close()}.
* (-stream) Streaming 'get' (returns InputStream); user must call {@link Session#close()}.
*/
STREAM("-stream");
STREAM("-stream"),
/**
* (-D) Delete the remote file after successful transfer (get, mget).
*/
DELETE("-D");
private String option;
@@ -324,8 +330,9 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
}
/**
* Specify the array of options for various gateway commands.
* Specify the options for various gateway commands as a space-delimited string.
* @param options the options to set
* @see Option
*/
public void setOptions(String options) {
Assert.hasText(options, "'options' must not be empty.");
@@ -341,6 +348,7 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
* Specify the array of options for various gateway commands.
* @param options the {@link Option} array to use.
* @since 5.0
* @see Option
*/
public void setOption(Option... options) {
Assert.notNull(options, "'options' must not be null");
@@ -1043,6 +1051,15 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
|| FileExistsMode.REPLACE_IF_MODIFIED.equals(fileExistsMode)) {
localFile.setLastModified(getModified(fileInfo));
}
if (this.options.contains(Option.DELETE)) {
boolean result = session.remove(remoteFilePath);
if (!result) {
logger.error("Failed to delete: " + remoteFilePath);
}
else if (logger.isDebugEnabled()) {
logger.debug(remoteFilePath + " deleted");
}
}
}
else if (FileExistsMode.REPLACE_IF_MODIFIED.equals(fileExistsMode)) {
if (logger.isDebugEnabled()) {

3
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml
View File

@@ -17,7 +17,8 @@
<int:channel id="inboundGet"/>
<int-ftp:outbound-gateway session-factory="ftpSessionFactory"
<int-ftp:outbound-gateway id="getGW"
session-factory="ftpSessionFactory"
request-channel="inboundGet"
command="get"
command-options="-P"

20
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java
View File

@@ -60,6 +60,7 @@ import org.mockito.Mockito;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.expression.spel.standard.SpelExpressionParser;
import org.springframework.integration.IntegrationMessageHeaderAccessor;
@@ -70,9 +71,11 @@ import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.file.remote.InputStreamCallback;
import org.springframework.integration.file.remote.MessageSessionCallback;
import org.springframework.integration.file.remote.RemoteFileTemplate;
import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway.Option;
import org.springframework.integration.file.remote.session.Session;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.ftp.FtpTestSupport;
import org.springframework.integration.ftp.gateway.FtpOutboundGateway;
import org.springframework.integration.ftp.session.FtpRemoteFileTemplate;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.integration.support.PartialSuccessException;
@@ -107,6 +110,10 @@ public class FtpServerOutboundTests extends FtpTestSupport {
@Autowired
private DirectChannel inboundGet;
@Autowired
@Qualifier("getGW.handler")
private FtpOutboundGateway getGw;
@Autowired
private DirectChannel invalidDirExpression;
@@ -181,6 +188,19 @@ public class FtpServerOutboundTests extends FtpTestSupport {
containsString(dir.toUpperCase()));
}
@Test
public void testGetWithRemove() {
String dir = "ftpSource/";
this.getGw.setOption(Option.DELETE);
this.inboundGet.send(new GenericMessage<Object>(dir + "ftpSource2.txt"));
Message<?> result = this.output.receive(1000);
assertNotNull(result);
File localFile = (File) result.getPayload();
assertThat(localFile.getPath().replaceAll(Matcher.quoteReplacement(File.separator), "/"),
containsString(dir.toUpperCase()));
assertThat(new File(getSourceRemoteDirectory(), "ftpSource2.txt").exists(), equalTo(false));
}
@Test
public void testInt2866InvalidLocalDirectoryExpression() {
try {

14
src/reference/asciidoc/ftp.adoc
View File

@@ -833,10 +833,13 @@ The result of the _nlst_ is just the names, therefore the framework can't determ
_get_ retrieves a remote file and supports the following option:
* -P - preserve the timestamp of the remote file
* -P - preserve the timestamp of the remote file.
* -stream - retrieve the remote file as a stream.
* -D - delete the remote file after successful transfer.
The remote file is NOT deleted if the transfer is ignored because the `FileExistsMode` is `IGNORE` and the local file already exists.
The remote directory is provided in the `file_remoteDirectory` header, and the filename is provided in the `file_remoteFile` header.
The message payload resulting from a _get_ operation is a `File` object representing the retrieved file, or
@@ -888,11 +891,14 @@ You can either do that in your custom code, or route a copy of the message to a
_mget_ retrieves multiple remote files based on a pattern and supports the following options:
* -P - preserve the timestamps of the remote files
* -P - preserve the timestamps of the remote files.
* -R - retrieve the entire directory tree recursively
* -R - retrieve the entire directory tree recursively.
* -x - Throw an exception if no files match the pattern (otherwise an empty list is returned)
* -x - Throw an exception if no files match the pattern (otherwise an empty list is returned).
* -D - delete each remote file after successful transfer.
The remote file is NOT deleted if the transfer is ignored because the `FileExistsMode` is `IGNORE` and the local file already exists.
The message payload resulting from an _mget_ operation is a `List<File>` object - a List of File objects, each representing a retrieved file.

14
src/reference/asciidoc/sftp.adoc
View File

@@ -855,10 +855,13 @@ The SFTP protocol doesn't provide _list names_ functionality, s this command is
_get_ retrieves a remote file and supports the following option:
* -P - preserve the timestamp of the remote file
* -P - preserve the timestamp of the remote file.
* -stream - retrieve the remote file as a stream.
* -D - delete the remote file after successful transfer.
The remote file is NOT deleted if the transfer is ignored because the `FileExistsMode` is `IGNORE` and the local file already exists.
The remote directory is provided in the `file_remoteDirectory` header, and the filename is provided in the `file_remoteFile` header.
The message payload resulting from a _get_ operation is a `File` object representing the retrieved file, or
@@ -910,11 +913,14 @@ You can either do that in your custom code, or route a copy of the message to a
_mget_ retrieves multiple remote files based on a pattern and supports the following options:
* -P - preserve the timestamps of the remote files
* -P - preserve the timestamps of the remote files.
* -R - retrieve the entire directory tree recursively
* -R - retrieve the entire directory tree recursively.
* -x - Throw an exception if no files match the pattern (otherwise an empty list is returned)
* -x - Throw an exception if no files match the pattern (otherwise an empty list is returned).
* -D - delete each remote file after successful transfer.
The remote file is NOT deleted if the transfer is ignored because the `FileExistsMode` is `IGNORE` and the local file already exists.
The message payload resulting from an _mget_ operation is a `List<File>` object - a List of File objects, each representing a retrieved file.

2
src/reference/asciidoc/whats-new.adoc
View File

@@ -176,6 +176,8 @@ The `RemoteFileTemplate` is supplied now with the `invoke(OperationsCallback<F,
New filters for detecting incomplete remote files are now provided.
The `FtpOutboundGateway` and `SftpOutboundGateway` now support an option to remove the remote file after a successful transfer using the `GET` or `MGET` commands.
See <<ftp>> and <<sftp>> for more information.
==== Integration Properties