INT-4015: Streaming Remote File Inbound Adapter

JIRA: https://jira.spring.io/browse/INT-4015 https://jira.spring.io/browse/INT-3854 Initial commit. Reworked to emit an input stream and use the file splitter. Add StreamTransformer. Add CLOSABLE_RESOURCE header so we can close the session automatically. Implement INT-3854, FTP, SFTP (S)FTP Namespace Changes Docs - also fixes a PDF overflow Polishing - PR Comments checkstyle fixes Polishing - Add Namespace for StreamParser Polishing - PR Comments
2016-04-27 15:10:02 -04:00
parent 6b6a38f8cb
commit 287d924fc0
61 changed files with 2703 additions and 918 deletions
--- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParser.java
+++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParser.java
@@ -17,6 +17,8 @@
 package org.springframework.integration.ftp.config;

 import org.springframework.integration.file.config.AbstractRemoteFileInboundChannelAdapterParser;
+import org.springframework.integration.file.filters.FileListFilter;
+import org.springframework.integration.file.remote.synchronizer.InboundFileSynchronizer;
 import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter;
 import org.springframework.integration.ftp.filters.FtpSimplePatternFileListFilter;
 import org.springframework.integration.ftp.inbound.FtpInboundFileSynchronizer;
@@ -37,18 +39,18 @@ public class FtpInboundChannelAdapterParser extends AbstractRemoteFileInboundCha
 	}

 	@Override
-	protected String getInboundFileSynchronizerClassname() {
-		return FtpInboundFileSynchronizer.class.getName();
+	protected Class<? extends InboundFileSynchronizer> getInboundFileSynchronizerClass() {
+		return FtpInboundFileSynchronizer.class;
 	}

 	@Override
-	protected String getSimplePatternFileListFilterClassname() {
-		return FtpSimplePatternFileListFilter.class.getName();
+	protected Class<? extends FileListFilter<?>> getSimplePatternFileListFilterClass() {
+		return FtpSimplePatternFileListFilter.class;
 	}

 	@Override
-	protected String getRegexPatternFileListFilterClassname() {
-		return FtpRegexPatternFileListFilter.class.getName();
+	protected Class<? extends FileListFilter<?>> getRegexPatternFileListFilterClass() {
+		return FtpRegexPatternFileListFilter.class;
 	}

 }
--- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpNamespaceHandler.java
+++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpNamespaceHandler.java
@@ -33,6 +33,8 @@ public class FtpNamespaceHandler extends AbstractIntegrationNamespaceHandler {
 	@Override
 	public void init() {
 		registerBeanDefinitionParser("inbound-channel-adapter", new FtpInboundChannelAdapterParser());
+		registerBeanDefinitionParser("inbound-streaming-channel-adapter",
+				new FtpStreamingInboundChannelAdapterParser());
 		registerBeanDefinitionParser("outbound-channel-adapter", new FtpOutboundChannelAdapterParser());
 		registerBeanDefinitionParser("outbound-gateway", new FtpOutboundGatewayParser());
 	}
--- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpStreamingInboundChannelAdapterParser.java
+++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpStreamingInboundChannelAdapterParser.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2016 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.integration.ftp.config;
+
+import org.springframework.integration.core.MessageSource;
+import org.springframework.integration.file.config.AbstractRemoteFileStreamingInboundChannelAdapterParser;
+import org.springframework.integration.file.filters.FileListFilter;
+import org.springframework.integration.file.remote.RemoteFileOperations;
+import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter;
+import org.springframework.integration.ftp.filters.FtpSimplePatternFileListFilter;
+import org.springframework.integration.ftp.inbound.FtpStreamingMessageSource;
+import org.springframework.integration.ftp.session.FtpRemoteFileTemplate;
+
+/**
+ * @author Gary Russell
+ * @since 4.3
+ *
+ */
+public class FtpStreamingInboundChannelAdapterParser extends AbstractRemoteFileStreamingInboundChannelAdapterParser {
+
+	@Override
+	protected Class<? extends RemoteFileOperations<?>> getTemplateClass() {
+		return FtpRemoteFileTemplate.class;
+	}
+
+	@Override
+	protected Class<? extends MessageSource<?>> getMessageSourceClass() {
+		return FtpStreamingMessageSource.class;
+	}
+
+	@Override
+	protected Class<? extends FileListFilter<?>> getSimplePatternFileListFilterClass() {
+		return FtpSimplePatternFileListFilter.class;
+	}
+
+	@Override
+	protected Class<? extends FileListFilter<?>> getRegexPatternFileListFilterClass() {
+		return FtpRegexPatternFileListFilter.class;
+	}
+
+}
--- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSource.java
+++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSource.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2016 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.integration.ftp.inbound;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.commons.net.ftp.FTPFile;
+
+import org.springframework.integration.file.remote.AbstractFileInfo;
+import org.springframework.integration.file.remote.AbstractRemoteFileStreamingMessageSource;
+import org.springframework.integration.file.remote.RemoteFileTemplate;
+import org.springframework.integration.ftp.session.FtpFileInfo;
+
+/**
+ * Message source for streaming FTP remote file contents.
+ *
+ * @author Gary Russell
+ * @since 4.3
+ *
+ */
+public class FtpStreamingMessageSource extends AbstractRemoteFileStreamingMessageSource<FTPFile> {
+
+	/**
+	 * Construct an instance with the supplied template.
+	 * @param template the template.
+	 */
+	public FtpStreamingMessageSource(RemoteFileTemplate<FTPFile> template) {
+		super(template, null);
+	}
+
+	/**
+	 * Construct an instance with the supplied template and comparator.
+	 * Note: the comparator is applied each time the remote directory is listed
+	 * which only occurs when the previous list is exhausted.
+	 * @param template the template.
+	 * @param comparator the comparator.
+	 */
+	public FtpStreamingMessageSource(RemoteFileTemplate<FTPFile> template,
+			Comparator<AbstractFileInfo<FTPFile>> comparator) {
+		super(template, comparator);
+	}
+
+	@Override
+	public String getComponentType() {
+		return "ftp:inbound-streaming-channel-adapter";
+	}
+
+	@Override
+	protected List<AbstractFileInfo<FTPFile>> asFileInfoList(Collection<FTPFile> files) {
+		List<AbstractFileInfo<FTPFile>> canonicalFiles = new ArrayList<AbstractFileInfo<FTPFile>>();
+		for (FTPFile file : files) {
+			canonicalFiles.add(new FtpFileInfo(file));
+		}
+		return canonicalFiles;
+	}
+
+}
--- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpSession.java
+++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpSession.java
@@ -146,7 +146,11 @@ public class FtpSession implements Session<FTPFile> {
 	public void close() {
 		try {
 			if (this.readingRaw.get()) {
-				finalizeRaw();
+				if (!finalizeRaw()) {
+					if (this.logger.isWarnEnabled()) {
+						this.logger.warn("Finalize on readRaw() returned false for " + this);
+					}
+				}
 			}
 			this.client.disconnect();
 		}
--- a/spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-4.3.xsd
+++ b/spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-4.3.xsd
@@ -24,7 +24,7 @@
 		</xsd:annotation>
 		<xsd:complexType>
 			<xsd:complexContent>
-				<xsd:extension base="base-ftp-adapter-type">
+				<xsd:extension base="base-outbound-adapter-type">
 					<xsd:all>
 						<xsd:element ref="integration:poller" minOccurs="0" maxOccurs="1"/>
 						<xsd:element name="request-handler-advice-chain" type="integration:handlerAdviceChainType" minOccurs="0" maxOccurs="1" />
@@ -78,38 +78,8 @@
 		</xsd:annotation>
 		<xsd:complexType>
 			<xsd:complexContent>
-				<xsd:extension base="base-ftp-adapter-type">
-					<xsd:sequence>
-						<xsd:element ref="integration:poller" minOccurs="0"
-							maxOccurs="1" />
-					</xsd:sequence>
-					<xsd:attribute name="channel" type="xsd:string">
-						<xsd:annotation>
-							<xsd:appinfo>
-								<tool:annotation kind="ref">
-									<tool:expected-type type="org.springframework.messaging.MessageChannel"/>
-								</tool:annotation>
-							</xsd:appinfo>
-							<xsd:documentation>
-								Identifies channel attached to this adapter. This channel where messages will be sent
-								to by this adapter.
-							</xsd:documentation>
-						</xsd:annotation>
-					</xsd:attribute>
-					<xsd:attribute name="filename-pattern" type="xsd:string">
-						<xsd:annotation>
-							<xsd:documentation>
-								Allows you to provide a file name pattern to
-								determine the file names
-								that need to be scanned.
-								This is based on
-								simple pattern matching (e.g., "*.txt, fo*.txt"
-								etc.)
-							</xsd:documentation>
-						</xsd:annotation>
-					</xsd:attribute>
-					<xsd:attribute name="local-filename-generator-expression"
-						type="xsd:string">
+				<xsd:extension base="base-inbound-adapter-type">
+					<xsd:attribute name="local-filename-generator-expression" type="xsd:string">
 						<xsd:annotation>
 							<xsd:documentation>
 								Allows you to provide a SpEL expression to
@@ -125,17 +95,6 @@
 							</xsd:documentation>
 						</xsd:annotation>
 					</xsd:attribute>
-					<xsd:attribute name="filename-regex" type="xsd:string">
-						<xsd:annotation>
-							<xsd:documentation>
-								Allows you to provide a Regular Expression to
-								determine the file names
-								that need to be scanned.
-								(e.g.,
-								"f[o]+\.txt" etc.)
-							</xsd:documentation>
-						</xsd:annotation>
-					</xsd:attribute>
 					<xsd:attribute name="comparator" type="xsd:string">
 						<xsd:annotation>
 							<xsd:documentation><![CDATA[
@@ -144,22 +103,6 @@
 							]]></xsd:documentation>
 						</xsd:annotation>
 					</xsd:attribute>
-					<xsd:attribute name="filter" type="xsd:string">
-						<xsd:annotation>
-							<xsd:appinfo>
-								<tool:annotation kind="ref">
-									<tool:expected-type
-										type="org.springframework.integration.file.filters.FileListFilter" />
-								</tool:annotation>
-							</xsd:appinfo>
-							<xsd:documentation>
-								Allows you to specify a reference to a
-								[org.springframework.integration.file.filters.FileListFilter]
-								bean. This filter is applied to files on the remote server and
-								only files that pass the filter are retrieved.
-							</xsd:documentation>
-						</xsd:annotation>
-					</xsd:attribute>
 					<xsd:attribute name="local-filter" type="xsd:string">
 						<xsd:annotation>
 							<xsd:appinfo>
@@ -220,16 +163,29 @@
 							</xsd:documentation>
 						</xsd:annotation>
 					</xsd:attribute>
-					<xsd:attribute name="remote-directory-expression"
-						type="xsd:string">
+					<xsd:attributeGroup ref="tempSuffixGroup" />
+				</xsd:extension>
+			</xsd:complexContent>
+		</xsd:complexType>
+	</xsd:element>
+
+	<xsd:element name="inbound-streaming-channel-adapter">
+		<xsd:annotation>
+			<xsd:documentation>
+				Configures a 'SourcePollingChannelAdapter' Endpoint for the
+				'org.springframework.integration.ftp.inbound.FtpInboundStreamingMessageSource'.
+			</xsd:documentation>
+		</xsd:annotation>
+		<xsd:complexType>
+			<xsd:complexContent>
+				<xsd:extension base="base-inbound-adapter-type">
+					<xsd:attribute name="comparator" type="xsd:string">
 						<xsd:annotation>
-							<xsd:documentation>
-								Specify a SpEL expression which
-								will be used to evaluate the directory
-								path from where the files will be transferred
-								(e.g., "@someBean.fetchDirectory");
-								Mutually exclusive with 'remote-directory'.
-							</xsd:documentation>
+							<xsd:documentation><![CDATA[
+								Specify a Comparator to be used when ordering Files. If none is provided, the
+								order in which files are processed is the order they are received from the
+								FTP server. The generic type of the Comparator must be 'FtpFileInfo'.
+							]]></xsd:documentation>
 						</xsd:annotation>
 					</xsd:attribute>
 				</xsd:extension>
@@ -247,7 +203,7 @@
 		</xsd:annotation>
 		<xsd:complexType>
 			<xsd:complexContent>
-				<xsd:extension base="base-ftp-adapter-type">
+				<xsd:extension base="base-outbound-adapter-type">
 					<xsd:all>
 						<xsd:element ref="integration:poller" minOccurs="0" maxOccurs="1" />
 						<xsd:element name="request-handler-advice-chain" type="integration:handlerAdviceChainType"
@@ -529,29 +485,63 @@
 		</xsd:complexType>
 	</xsd:element>

-	<xsd:complexType name="base-ftp-adapter-type">
+	<xsd:complexType name="base-inbound-adapter-type">
 		<xsd:complexContent>
 			<xsd:extension base="base-adapter-type">
-				<xsd:attribute name="remote-directory" type="xsd:string" use="optional">
+				<xsd:sequence>
+					<xsd:element ref="integration:poller" minOccurs="0"
+						maxOccurs="1" />
+				</xsd:sequence>
+				<xsd:attribute name="channel" type="xsd:string">
 					<xsd:annotation>
+						<xsd:appinfo>
+							<tool:annotation kind="ref">
+								<tool:expected-type type="org.springframework.messaging.MessageChannel"/>
+							</tool:annotation>
+						</xsd:appinfo>
 						<xsd:documentation>
-							Identifies the remote directory path (e.g., "/remote/mytransfers")
-							Mutually exclusive with 'remote-directory-expression'.
+							Identifies channel attached to this adapter.
+							The channel to which messages will be sent
+							by this adapter.
 						</xsd:documentation>
 					</xsd:annotation>
 				</xsd:attribute>
-				<xsd:attribute name="temporary-remote-directory" type="xsd:string"
-					use="optional">
+				<xsd:attribute name="filename-pattern" type="xsd:string">
 					<xsd:annotation>
 						<xsd:documentation>
-							Identifies the remote temporary directory path (e.g., "/remote/temp/mytransfers")
+							Allows you to provide a file name pattern to
+							determine the file names
+							that need to be scanned.
+							This is based on
+							simple pattern matching (e.g., "*.txt, fo*.txt"
+							etc.)
 						</xsd:documentation>
 					</xsd:annotation>
 				</xsd:attribute>
-				<xsd:attribute name="charset" type="xsd:string" default="UTF-8">
+				<xsd:attribute name="filename-regex" type="xsd:string">
 					<xsd:annotation>
 						<xsd:documentation>
-							Allows you to specify Charset (e.g., US-ASCII, ISO-8859-1, UTF-8). [UTF-8] is default
+							Allows you to provide a Regular Expression to
+							determine the file names
+							that need to be scanned.
+							(e.g.,
+							"f[o]+\.txt" etc.)
+						</xsd:documentation>
+					</xsd:annotation>
+				</xsd:attribute>
+				<xsd:attribute name="filter" type="xsd:string">
+					<xsd:annotation>
+						<xsd:appinfo>
+							<tool:annotation kind="ref">
+								<tool:expected-type
+									type="org.springframework.integration.file.filters.FileListFilter" />
+							</tool:annotation>
+						</xsd:appinfo>
+						<xsd:documentation>
+							Allows you to specify a reference to a
+							[org.springframework.integration.file.filters.FileListFilter]
+							bean. This filter is applied to files on the remote server and
+							only files that pass the filter are retrieved.
 						</xsd:documentation>
 					</xsd:annotation>
 				</xsd:attribute>
@@ -559,10 +549,24 @@
 		</xsd:complexContent>
 	</xsd:complexType>

+	<xsd:complexType name="base-outbound-adapter-type">
+		<xsd:complexContent>
+			<xsd:extension base="base-adapter-type">
+				<xsd:attribute name="temporary-remote-directory" type="xsd:string" use="optional">
+					<xsd:annotation>
+						<xsd:documentation>
+							Identifies the remote temporary directory path (e.g., "/remote/temp/mytransfers")
+						</xsd:documentation>
+					</xsd:annotation>
+				</xsd:attribute>
+				<xsd:attributeGroup ref="tempSuffixGroup" />
+			</xsd:extension>
+		</xsd:complexContent>
+	</xsd:complexType>
+
 	<xsd:complexType name="base-adapter-type">
 		<xsd:attribute name="id" type="xsd:string" />
-		<xsd:attribute name="session-factory" type="xsd:string"
-			use="required">
+		<xsd:attribute name="session-factory" type="xsd:string" use="required">
 			<xsd:annotation>
 				<xsd:appinfo>
 					<tool:annotation kind="ref">
@@ -576,17 +580,6 @@
 				]]></xsd:documentation>
 			</xsd:annotation>
 		</xsd:attribute>
-		<xsd:attribute name="temporary-file-suffix" type="xsd:string">
-			<xsd:annotation>
-				<xsd:documentation>
-					Extension used when downloading files. We
-					change
-					it right after we know it's
-					downloaded.
-				</xsd:documentation>
-			</xsd:annotation>
-		</xsd:attribute>
-
 		<xsd:attribute name="remote-file-separator" type="xsd:string"
 			default="/">
 			<xsd:annotation>
@@ -597,7 +590,39 @@
 				</xsd:documentation>
 			</xsd:annotation>
 		</xsd:attribute>
+		<xsd:attribute name="remote-directory" type="xsd:string" use="optional">
+			<xsd:annotation>
+				<xsd:documentation>
+					Identifies the remote directory path (e.g., "/remote/mytransfers")
+					Mutually exclusive with 'remote-directory-expression'.
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+		<xsd:attribute name="remote-directory-expression"
+			type="xsd:string">
+			<xsd:annotation>
+				<xsd:documentation>
+					Specify a SpEL expression which
+					will be used to evaluate the directory
+					path to where the files will be transferred
+					(e.g., "headers.['remote_dir'] + '/myTransfers'" for outbound endpoints)
+					There is no root object (message) for inbound endpoints
+					(e.g., "@someBean.fetchDirectory");
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
 		<xsd:attributeGroup ref="integration:smartLifeCycleAttributeGroup"/>
 	</xsd:complexType>

+	<xsd:attributeGroup name="tempSuffixGroup">
+		<xsd:attribute name="temporary-file-suffix" type="xsd:string">
+			<xsd:annotation>
+				<xsd:documentation>
+					Extension used when downloading files. We change
+					it right after we know it's downloaded.
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+	</xsd:attributeGroup>
+
 </xsd:schema>