GH-3043: Add FileHeaders.REMOTE_HOST header (#3044)

* GH-3043: Add FileHeaders.REMOTE_HOST header

Fixes https://github.com/spring-projects/spring-integration/issues/3043

* Populate a `FileHeaders.REMOTE_HOST` from the
`AbstractRemoteFileStreamingMessageSource` and "get"-based commands
in the `AbstractRemoteFileOutboundGateway`
* Extract the value from the a `Session.getHost()` contract
* The `AbstractInboundFileSynchronizingMessageSource` cannot be
addressed with this because the real message is already based on the
locally stored file
* Adjust some affected tests according our code style requirements

* * Add remote file info support into `AbstractInboundFileSynchronizingMessageSource`
* Introduce a `MetadataStore` functionality into the `AbstractInboundFileSynchronizer`
to gather a remote file info an save it in the URI style against local file
* Retrieve such an info in the `AbstractInboundFileSynchronizingMessageSource`
during local file polling
* Introduce `protocol()` contract for the `AbstractInboundFileSynchronizer`
to build a proper URI in the metadata for external readers to distinguish
remote files properly
* Document the feature

* * Fix some typos in Docs

* * Rename property and header constant to the `HOST_PORT` pair
* Fix typos in Docs
* Add  `remote-file-metadata-store` and `metadata-store-prefix` into XSD
of (S)FTP Inbound Channel Adapters
* Add `remoteFileMetadataStore` and `metadataStorePrefix` options
into `RemoteFileInboundChannelAdapterSpec` for Java DSL

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/inbound/SftpInboundFileSynchronizer.java

@@ -59,4 +59,9 @@ public class SftpInboundFileSynchronizer extends AbstractInboundFileSynchronizer
 		return (long) file.getAttrs().getMTime() * 1000;
 	}
 
+	@Override
+	protected String protocol() {
+		return "sftp";
+	}
+
 }

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/session/SftpSession.java

@@ -46,16 +46,18 @@ import com.jcraft.jsch.SftpException;
  * @author Mark Fisher
  * @author Oleg Zhurakousky
  * @author Gary Russell
+ * @author Artem Bilan
+ *
  * @since 2.0
  */
 public class SftpSession implements Session<LsEntry> {
 
+	private static final Log LOGGER = LogFactory.getLog(SftpSession.class);
+
 	private static final String SESSION_IS_NOT_CONNECTED = "session is not connected";
 
 	private static final Duration DEFAULT_CHANNEL_CONNECT_TIMEOUT = Duration.ofSeconds(5);
 
-	private final Log logger = LogFactory.getLog(this.getClass());
-
 	private final com.jcraft.jsch.Session jschSession;
 
 	private final JSchSessionWrapper wrapper;
@@ -214,14 +216,14 @@ public class SftpSession implements Session<LsEntry> {
 			this.channel.rename(pathFrom, pathTo);
 		}
 		catch (SftpException sftpex) {
-			if (this.logger.isDebugEnabled()) {
-				this.logger.debug("Initial File rename failed, possibly because file already exists. Will attempt to delete file: "
-						+ pathTo + " and execute rename again.");
+			if (LOGGER.isDebugEnabled()) {
+				LOGGER.debug("Initial File rename failed, possibly because file already exists. " +
+						"Will attempt to delete file: " + pathTo + " and execute rename again.");
 			}
 			try {
-				this.remove(pathTo);
-				if (this.logger.isDebugEnabled()) {
-					this.logger.debug("Delete file: " + pathTo + " succeeded. Will attempt rename again");
+				remove(pathTo);
+				if (LOGGER.isDebugEnabled()) {
+					LOGGER.debug("Delete file: " + pathTo + " succeeded. Will attempt rename again");
 				}
 			}
 			catch (IOException ioex) {
@@ -240,8 +242,8 @@ public class SftpSession implements Session<LsEntry> {
 				throw exception; // NOSONAR - added to suppressed exceptions
 			}
 		}
-		if (this.logger.isDebugEnabled()) {
-			this.logger.debug("File: " + pathFrom + " was successfully renamed to " + pathTo);
+		if (LOGGER.isDebugEnabled()) {
+			LOGGER.debug("File: " + pathFrom + " was successfully renamed to " + pathTo);
 		}
 	}
 
@@ -300,6 +302,11 @@ public class SftpSession implements Session<LsEntry> {
 		return this.channel;
 	}
 
+	@Override
+	public String getHostPort() {
+		return this.jschSession.getHost() + ':' + this.jschSession.getPort();
+	}
+
 	@Override
 	public boolean test() {
 		return isOpen() && doTest();

spring-integration-sftp/src/main/resources/org/springframework/integration/sftp/config/spring-integration-sftp-5.2.xsd

@@ -180,6 +180,29 @@
 							</xsd:documentation>
 						</xsd:annotation>
 					</xsd:attribute>
+					<xsd:attribute name="remote-file-metadata-store" type="xsd:string">
+						<xsd:annotation>
+							<xsd:appinfo>
+								<tool:annotation kind="ref">
+									<tool:expected-type
+											type="org.springframework.integration.metadata.MetadataStore" />
+								</tool:annotation>
+							</xsd:appinfo>
+							<xsd:documentation>
+								Reference to a MetadataStore for saving remote files information between
+								synchronization and polling.
+							</xsd:documentation>
+						</xsd:annotation>
+					</xsd:attribute>
+					<xsd:attribute name="metadata-store-prefix" type="xsd:string">
+						<xsd:annotation>
+							<xsd:documentation>
+								Specify a prefix for metadata store to distinguish keys from another places
+								where the same shared store is used.
+								By default, the remote a component name is used.
+							</xsd:documentation>
+						</xsd:annotation>
+					</xsd:attribute>
 					<xsd:attributeGroup ref="tempSuffixGroup" />
 				</xsd:extension>
 			</xsd:complexContent>

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests-context.xml

@@ -45,12 +45,16 @@
 								  max-fetch-size="42"
 								  delete-remote-files="${delete.remote.files}"
 								  auto-startup="false"
-								  preserve-timestamp="true">
+								  preserve-timestamp="true"
+								  remote-file-metadata-store="metadataStore"
+								  metadata-store-prefix="testPrefix">
 		<poller fixed-rate="1000">
 			<transactional synchronization-factory="syncFactory"/>
 		</poller>
 	</sftp:inbound-channel-adapter>
 
+	<beans:bean id="metadataStore" class="org.springframework.integration.metadata.SimpleMetadataStore"/>
+
 	<beans:bean id="acceptAllFilter" class="org.springframework.integration.file.filters.AcceptAllFileListFilter"/>
 
 	<transaction-synchronization-factory id="syncFactory">

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests.java

@@ -17,6 +17,7 @@
 package org.springframework.integration.sftp.config;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 
 import java.io.File;
 import java.util.Collection;
@@ -34,6 +35,7 @@ import org.springframework.context.support.ClassPathXmlApplicationContext;
 import org.springframework.expression.Expression;
 import org.springframework.integration.endpoint.SourcePollingChannelAdapter;
 import org.springframework.integration.file.filters.FileListFilter;
+import org.springframework.integration.metadata.MetadataStore;
 import org.springframework.integration.sftp.inbound.SftpInboundFileSynchronizer;
 import org.springframework.integration.sftp.inbound.SftpInboundFileSynchronizingMessageSource;
 import org.springframework.integration.test.util.TestUtils;
@@ -44,6 +46,7 @@ import org.springframework.messaging.PollableChannel;
  * @author Oleg Zhurakousky
  * @author Gary Russell
  * @author Gunnar Hillert
+ * @author Artem Bilan
  */
 public class InboundChannelAdapterParserTests {
 
@@ -53,9 +56,9 @@ public class InboundChannelAdapterParserTests {
 	}
 
 	@Test
-	public void testAutoStartup() throws Exception {
+	public void testAutoStartup() {
 		ConfigurableApplicationContext context =
-			new ClassPathXmlApplicationContext("SftpInboundAutostartup-context.xml", this.getClass());
+				new ClassPathXmlApplicationContext("SftpInboundAutostartup-context.xml", this.getClass());
 
 		SourcePollingChannelAdapter adapter = context.getBean("sftpAutoStartup", SourcePollingChannelAdapter.class);
 		assertThat(adapter.isRunning()).isFalse();
@@ -63,15 +66,15 @@ public class InboundChannelAdapterParserTests {
 	}
 
 	@Test
-	public void testWithLocalFiles() throws Exception {
+	public void testWithLocalFiles() {
 		ConfigurableApplicationContext context =
-			new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context.xml", this.getClass());
+				new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context.xml", this.getClass());
 		assertThat(new File("src/main/resources").exists()).isTrue();
 
 		Object adapter = context.getBean("sftpAdapterAutoCreate");
 		assertThat(adapter instanceof SourcePollingChannelAdapter).isTrue();
 		SftpInboundFileSynchronizingMessageSource source =
-			(SftpInboundFileSynchronizingMessageSource) TestUtils.getPropertyValue(adapter, "source");
+				(SftpInboundFileSynchronizingMessageSource) TestUtils.getPropertyValue(adapter, "source");
 		assertThat(source).isNotNull();
 
 		PriorityBlockingQueue<?> blockingQueue =
@@ -87,6 +90,10 @@ public class InboundChannelAdapterParserTests {
 		assertThat(TestUtils.getPropertyValue(synchronizer, "preserveTimestamp", Boolean.class)).isTrue();
 		String remoteFileSeparator = (String) TestUtils.getPropertyValue(synchronizer, "remoteFileSeparator");
 		assertThat(TestUtils.getPropertyValue(synchronizer, "temporaryFileSuffix", String.class)).isEqualTo(".bar");
+		assertThat(TestUtils.getPropertyValue(synchronizer, "remoteFileMetadataStore"))
+				.isSameAs(context.getBean(MetadataStore.class));
+		assertThat(TestUtils.getPropertyValue(synchronizer, "metadataStorePrefix", String.class))
+				.isEqualTo("testPrefix");
 		assertThat(remoteFileSeparator).isNotNull();
 		assertThat(remoteFileSeparator).isEqualTo(".");
 		PollableChannel requestChannel = context.getBean("requestChannel", PollableChannel.class);
@@ -103,7 +110,7 @@ public class InboundChannelAdapterParserTests {
 	@Test
 	public void testAutoChannel() {
 		ConfigurableApplicationContext context =
-			new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context.xml", this.getClass());
+				new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context.xml", this.getClass());
 		// Auto-created channel
 		MessageChannel autoChannel = context.getBean("autoChannel", MessageChannel.class);
 		SourcePollingChannelAdapter autoChannelAdapter = context.getBean("autoChannel.adapter",
@@ -117,30 +124,36 @@ public class InboundChannelAdapterParserTests {
 		context.close();
 	}
 
-	@Test(expected = BeanDefinitionStoreException.class)
+	@Test
 	//exactly one of 'filename-pattern' or 'filter' is allowed on SFTP inbound adapter
-	public void testFailWithFilePatternAndFilter() throws Exception {
+	public void testFailWithFilePatternAndFilter() {
 		assertThat(!new File("target/bar").exists()).isTrue();
-		new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context-fail.xml", this.getClass()).close();
+		assertThatExceptionOfType(BeanDefinitionStoreException.class)
+				.isThrownBy(() ->
+						new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context-fail.xml",
+								getClass()));
 	}
 
 	@Test
-	public void testLocalDirAutoCreated() throws Exception {
+	public void testLocalDirAutoCreated() {
 		assertThat(new File("foo").exists()).isFalse();
 		ConfigurableApplicationContext context = new ClassPathXmlApplicationContext(
-				"InboundChannelAdapterParserTests-context.xml", this.getClass());
+				"InboundChannelAdapterParserTests-context.xml", getClass());
 		assertThat(new File("foo").exists()).isTrue();
 		context.close();
 	}
 
-	@Test(expected = BeanCreationException.class)
-	public void testLocalDirAutoCreateFailed() throws Exception {
-		new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context-fail-autocreate.xml",
-				this.getClass()).close();
+	@Test
+	public void testLocalDirAutoCreateFailed() {
+		assertThatExceptionOfType(BeanCreationException.class)
+				.isThrownBy(() ->
+						new ClassPathXmlApplicationContext(
+								"InboundChannelAdapterParserTests-context-fail-autocreate.xml",
+								getClass()));
 	}
 
 	@After
-	public void cleanUp() throws Exception {
+	public void cleanUp() {
 		new File("foo").delete();
 	}
 

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/dsl/SftpTests.java

@@ -118,6 +118,7 @@ public class SftpTests extends SftpTestSupport {
 		assertThat(message).isNotNull();
 		assertThat(message.getPayload()).isInstanceOf(InputStream.class);
 		assertThat(message.getHeaders().get(FileHeaders.REMOTE_FILE)).isIn(" sftpSource1.txt", "sftpSource2.txt");
+		assertThat(message.getHeaders().get(FileHeaders.REMOTE_HOST_PORT, String.class)).contains("localhost:");
 		((InputStream) message.getPayload()).close();
 		new IntegrationMessageHeaderAccessor(message).getCloseableResource().close();
 

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpInboundRemoteFileSystemSynchronizerTests.java

@@ -36,6 +36,7 @@ import org.junit.Before;
 import org.junit.Test;
 
 import org.springframework.beans.factory.BeanFactory;
+import org.springframework.integration.file.FileHeaders;
 import org.springframework.integration.file.filters.AcceptOnceFileListFilter;
 import org.springframework.integration.file.filters.CompositeFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
@@ -58,6 +59,7 @@ import com.jcraft.jsch.SftpATTRS;
  * @author Gunnar Hillert
  * @author Gary Russell
  * @author Artem Bilan
+ *
  * @since 2.0
  */
 public class SftpInboundRemoteFileSystemSynchronizerTests {
@@ -97,7 +99,7 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
 		store.afterPropertiesSet();
 		SftpPersistentAcceptOnceFileListFilter persistFilter =
 				new SftpPersistentAcceptOnceFileListFilter(store, "foo");
-		List<FileListFilter<LsEntry>> filters = new ArrayList<FileListFilter<LsEntry>>();
+		List<FileListFilter<LsEntry>> filters = new ArrayList<>();
 		filters.add(persistFilter);
 		filters.add(patternFilter);
 		CompositeFileListFilter<LsEntry> filter = new CompositeFileListFilter<LsEntry>(filters);
@@ -109,27 +111,30 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
 		ms.setAutoCreateLocalDirectory(true);
 		ms.setLocalDirectory(localDirectory);
 		ms.setBeanFactory(mock(BeanFactory.class));
-		CompositeFileListFilter<File> localFileListFilter = new CompositeFileListFilter<File>();
+		CompositeFileListFilter<File> localFileListFilter = new CompositeFileListFilter<>();
 		localFileListFilter.addFilter(new RegexPatternFileListFilter(".*\\.test$"));
-		AcceptOnceFileListFilter<File> localAcceptOnceFilter = new AcceptOnceFileListFilter<File>();
+		AcceptOnceFileListFilter<File> localAcceptOnceFilter = new AcceptOnceFileListFilter<>();
 		localFileListFilter.addFilter(localAcceptOnceFilter);
 		ms.setLocalFilter(localFileListFilter);
 		ms.afterPropertiesSet();
 		ms.start();
 
-		Message<File> atestFile =  ms.receive();
+		Message<File> atestFile = ms.receive();
 		assertThat(atestFile).isNotNull();
 		assertThat(atestFile.getPayload().getName()).isEqualTo("a.test");
 		// The test remote files are created with the current timestamp + 1 day.
 		assertThat(atestFile.getPayload().lastModified()).isGreaterThan(System.currentTimeMillis());
 
-		Message<File> btestFile =  ms.receive();
+		assertThat(atestFile.getHeaders())
+				.containsKeys(FileHeaders.REMOTE_HOST_PORT, FileHeaders.REMOTE_DIRECTORY, FileHeaders.REMOTE_FILE);
+
+		Message<File> btestFile = ms.receive();
 		assertThat(btestFile).isNotNull();
 		assertThat(btestFile.getPayload().getName()).isEqualTo("b.test");
 		// The test remote files are created with the current timestamp + 1 day.
 		assertThat(atestFile.getPayload().lastModified()).isGreaterThan(System.currentTimeMillis());
 
-		Message<File> nothing =  ms.receive();
+		Message<File> nothing = ms.receive();
 		assertThat(nothing).isNull();
 
 		// two times because on the third receive (above) the internal queue will be empty, so it will attempt
@@ -143,7 +148,7 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
 		new File("test/a.test").delete();
 		new File("test/b.test").delete();
 		// the remote filter should prevent a re-fetch
-		nothing =  ms.receive();
+		nothing = ms.receive();
 		assertThat(nothing).isNull();
 
 		ms.stop();
@@ -153,7 +158,7 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
 
 	public static class TestSftpSessionFactory extends DefaultSftpSessionFactory {
 
-		private final Vector<LsEntry> sftpEntries = new Vector<LsEntry>();
+		private final Vector<LsEntry> sftpEntries = new Vector<>();
 
 		private void init() {
 			String[] files = new File("remote-test-dir").list();

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSourceTests.java

@@ -119,6 +119,7 @@ public class SftpStreamingMessageSourceTests extends SftpTestSupport {
 		received = (Message<byte[]>) this.data.receive(10000);
 		assertThat(received).isNotNull();
 		assertThat(received.getHeaders().get(FileHeaders.REMOTE_FILE_INFO)).isInstanceOf(SftpFileInfo.class);
+		assertThat(received.getHeaders().get(FileHeaders.REMOTE_HOST_PORT, String.class)).contains("localhost:");
 		this.adapter.stop();
 	}