From 5216dc2b023279f1e18aa070804deebb79263d1f Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Fri, 23 Jun 2017 15:38:03 -0400 Subject: [PATCH] INT-4304 (S)FTP Outbound Gateway Delete After GET JIRA: https://jira.spring.io/browse/INT-4304 Add an option to delete remote file(s) after GET/MGET. Polishing - log error on delete failure. --- .../AbstractRemoteFileOutboundGateway.java | 55 ++++++++++++------- .../FtpServerOutboundTests-context.xml | 3 +- .../ftp/outbound/FtpServerOutboundTests.java | 20 +++++++ src/reference/asciidoc/ftp.adoc | 14 +++-- src/reference/asciidoc/sftp.adoc | 14 +++-- src/reference/asciidoc/whats-new.adoc | 2 + 6 files changed, 80 insertions(+), 28 deletions(-) diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java index 99f28547ab..4bd49244c3 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java @@ -79,42 +79,42 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply public enum Command { /** - * List remote files. + * (ls) List remote files. */ LS("ls"), /** - * List remote file names. + * (nlst) List remote file names. */ NLST("nlst"), /** - * Retrieve a remote file. + * (get) Retrieve a remote file. */ GET("get"), /** - * Remove a remote file (path - including wildcards). + * (rm) Remove a remote file (path - including wildcards). */ RM("rm"), /** - * Retrieve multiple files matching a wildcard path. + * (mget) Retrieve multiple files matching a wildcard path. */ MGET("mget"), /** - * Move (rename) a remote file. + * (mv) Move (rename) a remote file. */ MV("mv"), /** - * Put a local file to the remote system. + * (put) Put a local file to the remote system. */ PUT("put"), /** - * Put multiple local files to the remote system. + * (mput) Put multiple local files to the remote system. */ MPUT("mput"); @@ -146,49 +146,55 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply public enum Option { /** - * Don't return full file information; just the name (ls). + * (-1) Don't return full file information; just the name (ls). */ NAME_ONLY("-1"), /** - * Include files beginning with {@code .}, including directories {@code .} and {@code ..} in the results (ls). + * (-a) Include files beginning with {@code .}, including directories {@code .} + * and {@code ..} in the results (ls). */ ALL("-a"), /** - * Do not sort the results (ls with NAME_ONLY). + * (-f) Do not sort the results (ls with NAME_ONLY). */ NOSORT("-f"), /** - * Include directories in the results (ls). + * (-dirs) Include directories in the results (ls). */ SUBDIRS("-dirs"), /** - * Include links in the results (ls). + * (-links) Include links in the results (ls). */ LINKS("-links"), /** - * Preserve the server timestamp (get, mget). + * (-P) Preserve the server timestamp (get, mget). */ PRESERVE_TIMESTAMP("-P"), /** - * Throw an exception if no files returned (mget). + * (-x) Throw an exception if no files returned (mget). */ EXCEPTION_WHEN_EMPTY("-x"), /** - * Recursive (ls, mget) + * (-R) Recursive (ls, mget) */ RECURSIVE("-R"), /** - * Streaming 'get' (returns InputStream); user must call {@link Session#close()}. + * (-stream) Streaming 'get' (returns InputStream); user must call {@link Session#close()}. */ - STREAM("-stream"); + STREAM("-stream"), + + /** + * (-D) Delete the remote file after successful transfer (get, mget). + */ + DELETE("-D"); private String option; @@ -324,8 +330,9 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply } /** - * Specify the array of options for various gateway commands. + * Specify the options for various gateway commands as a space-delimited string. * @param options the options to set + * @see Option */ public void setOptions(String options) { Assert.hasText(options, "'options' must not be empty."); @@ -341,6 +348,7 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply * Specify the array of options for various gateway commands. * @param options the {@link Option} array to use. * @since 5.0 + * @see Option */ public void setOption(Option... options) { Assert.notNull(options, "'options' must not be null"); @@ -1043,6 +1051,15 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply || FileExistsMode.REPLACE_IF_MODIFIED.equals(fileExistsMode)) { localFile.setLastModified(getModified(fileInfo)); } + if (this.options.contains(Option.DELETE)) { + boolean result = session.remove(remoteFilePath); + if (!result) { + logger.error("Failed to delete: " + remoteFilePath); + } + else if (logger.isDebugEnabled()) { + logger.debug(remoteFilePath + " deleted"); + } + } } else if (FileExistsMode.REPLACE_IF_MODIFIED.equals(fileExistsMode)) { if (logger.isDebugEnabled()) { diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml index 945141eda9..62be8b97aa 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml @@ -17,7 +17,8 @@ - (dir + "ftpSource2.txt")); + Message result = this.output.receive(1000); + assertNotNull(result); + File localFile = (File) result.getPayload(); + assertThat(localFile.getPath().replaceAll(Matcher.quoteReplacement(File.separator), "/"), + containsString(dir.toUpperCase())); + assertThat(new File(getSourceRemoteDirectory(), "ftpSource2.txt").exists(), equalTo(false)); + } + @Test public void testInt2866InvalidLocalDirectoryExpression() { try { diff --git a/src/reference/asciidoc/ftp.adoc b/src/reference/asciidoc/ftp.adoc index 818d55fd06..0855dccb6f 100644 --- a/src/reference/asciidoc/ftp.adoc +++ b/src/reference/asciidoc/ftp.adoc @@ -833,10 +833,13 @@ The result of the _nlst_ is just the names, therefore the framework can't determ _get_ retrieves a remote file and supports the following option: -* -P - preserve the timestamp of the remote file +* -P - preserve the timestamp of the remote file. * -stream - retrieve the remote file as a stream. +* -D - delete the remote file after successful transfer. +The remote file is NOT deleted if the transfer is ignored because the `FileExistsMode` is `IGNORE` and the local file already exists. + The remote directory is provided in the `file_remoteDirectory` header, and the filename is provided in the `file_remoteFile` header. The message payload resulting from a _get_ operation is a `File` object representing the retrieved file, or @@ -888,11 +891,14 @@ You can either do that in your custom code, or route a copy of the message to a _mget_ retrieves multiple remote files based on a pattern and supports the following options: -* -P - preserve the timestamps of the remote files +* -P - preserve the timestamps of the remote files. -* -R - retrieve the entire directory tree recursively +* -R - retrieve the entire directory tree recursively. -* -x - Throw an exception if no files match the pattern (otherwise an empty list is returned) +* -x - Throw an exception if no files match the pattern (otherwise an empty list is returned). + +* -D - delete each remote file after successful transfer. +The remote file is NOT deleted if the transfer is ignored because the `FileExistsMode` is `IGNORE` and the local file already exists. The message payload resulting from an _mget_ operation is a `List` object - a List of File objects, each representing a retrieved file. diff --git a/src/reference/asciidoc/sftp.adoc b/src/reference/asciidoc/sftp.adoc index f246037e4d..8322ca2e8a 100644 --- a/src/reference/asciidoc/sftp.adoc +++ b/src/reference/asciidoc/sftp.adoc @@ -855,10 +855,13 @@ The SFTP protocol doesn't provide _list names_ functionality, s this command is _get_ retrieves a remote file and supports the following option: -* -P - preserve the timestamp of the remote file +* -P - preserve the timestamp of the remote file. * -stream - retrieve the remote file as a stream. +* -D - delete the remote file after successful transfer. +The remote file is NOT deleted if the transfer is ignored because the `FileExistsMode` is `IGNORE` and the local file already exists. + The remote directory is provided in the `file_remoteDirectory` header, and the filename is provided in the `file_remoteFile` header. The message payload resulting from a _get_ operation is a `File` object representing the retrieved file, or @@ -910,11 +913,14 @@ You can either do that in your custom code, or route a copy of the message to a _mget_ retrieves multiple remote files based on a pattern and supports the following options: -* -P - preserve the timestamps of the remote files +* -P - preserve the timestamps of the remote files. -* -R - retrieve the entire directory tree recursively +* -R - retrieve the entire directory tree recursively. -* -x - Throw an exception if no files match the pattern (otherwise an empty list is returned) +* -x - Throw an exception if no files match the pattern (otherwise an empty list is returned). + +* -D - delete each remote file after successful transfer. +The remote file is NOT deleted if the transfer is ignored because the `FileExistsMode` is `IGNORE` and the local file already exists. The message payload resulting from an _mget_ operation is a `List` object - a List of File objects, each representing a retrieved file. diff --git a/src/reference/asciidoc/whats-new.adoc b/src/reference/asciidoc/whats-new.adoc index 04ebbf74f9..6ca4e47439 100644 --- a/src/reference/asciidoc/whats-new.adoc +++ b/src/reference/asciidoc/whats-new.adoc @@ -176,6 +176,8 @@ The `RemoteFileTemplate` is supplied now with the `invoke(OperationsCallback> and <> for more information. ==== Integration Properties