Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-3043: Add FileHeaders.REMOTE_HOST header (#3044)

Browse Source 
* GH-3043: Add FileHeaders.REMOTE_HOST header

Fixes https://github.com/spring-projects/spring-integration/issues/3043

* Populate a `FileHeaders.REMOTE_HOST` from the
`AbstractRemoteFileStreamingMessageSource` and "get"-based commands
in the `AbstractRemoteFileOutboundGateway`
* Extract the value from the a `Session.getHost()` contract
* The `AbstractInboundFileSynchronizingMessageSource` cannot be
addressed with this because the real message is already based on the
locally stored file
* Adjust some affected tests according our code style requirements

* * Add remote file info support into `AbstractInboundFileSynchronizingMessageSource`
* Introduce a `MetadataStore` functionality into the `AbstractInboundFileSynchronizer`
to gather a remote file info an save it in the URI style against local file
* Retrieve such an info in the `AbstractInboundFileSynchronizingMessageSource`
during local file polling
* Introduce `protocol()` contract for the `AbstractInboundFileSynchronizer`
to build a proper URI in the metadata for external readers to distinguish
remote files properly
* Document the feature

* * Fix some typos in Docs

* * Rename property and header constant to the `HOST_PORT` pair
* Fix typos in Docs
* Add  `remote-file-metadata-store` and `metadata-store-prefix` into XSD
of (S)FTP Inbound Channel Adapters
* Add `remoteFileMetadataStore` and `metadataStorePrefix` options
into `RemoteFileInboundChannelAdapterSpec` for Java DSL
This commit is contained in:
Artem Bilan
2019-08-28 08:58:14 -04:00
committed by Gary Russell
parent ff15d5265d
commit a756e6334d
33 changed files with 605 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests-context.xml
View File

@@ -45,12 +45,16 @@
max-fetch-size="42"
delete-remote-files="${delete.remote.files}"
auto-startup="false"
preserve-timestamp="true">
preserve-timestamp="true"
remote-file-metadata-store="metadataStore"
metadata-store-prefix="testPrefix">
<poller fixed-rate="1000">
<transactional synchronization-factory="syncFactory"/>
</poller>
</sftp:inbound-channel-adapter>
<beans:bean id="metadataStore" class="org.springframework.integration.metadata.SimpleMetadataStore"/>
<beans:bean id="acceptAllFilter" class="org.springframework.integration.file.filters.AcceptAllFileListFilter"/>
<transaction-synchronization-factory id="syncFactory">

45
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests.java
View File

@@ -17,6 +17,7 @@
package org.springframework.integration.sftp.config;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import java.io.File;
import java.util.Collection;
@@ -34,6 +35,7 @@ import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.expression.Expression;
import org.springframework.integration.endpoint.SourcePollingChannelAdapter;
import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.metadata.MetadataStore;
import org.springframework.integration.sftp.inbound.SftpInboundFileSynchronizer;
import org.springframework.integration.sftp.inbound.SftpInboundFileSynchronizingMessageSource;
import org.springframework.integration.test.util.TestUtils;
@@ -44,6 +46,7 @@ import org.springframework.messaging.PollableChannel;
* @author Oleg Zhurakousky
* @author Gary Russell
* @author Gunnar Hillert
* @author Artem Bilan
*/
public class InboundChannelAdapterParserTests {
@@ -53,9 +56,9 @@ public class InboundChannelAdapterParserTests {
}
@Test
public void testAutoStartup() throws Exception {
public void testAutoStartup() {
ConfigurableApplicationContext context =
new ClassPathXmlApplicationContext("SftpInboundAutostartup-context.xml", this.getClass());
new ClassPathXmlApplicationContext("SftpInboundAutostartup-context.xml", this.getClass());
SourcePollingChannelAdapter adapter = context.getBean("sftpAutoStartup", SourcePollingChannelAdapter.class);
assertThat(adapter.isRunning()).isFalse();
@@ -63,15 +66,15 @@ public class InboundChannelAdapterParserTests {
}
@Test
public void testWithLocalFiles() throws Exception {
public void testWithLocalFiles() {
ConfigurableApplicationContext context =
new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context.xml", this.getClass());
new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context.xml", this.getClass());
assertThat(new File("src/main/resources").exists()).isTrue();
Object adapter = context.getBean("sftpAdapterAutoCreate");
assertThat(adapter instanceof SourcePollingChannelAdapter).isTrue();
SftpInboundFileSynchronizingMessageSource source =
(SftpInboundFileSynchronizingMessageSource) TestUtils.getPropertyValue(adapter, "source");
(SftpInboundFileSynchronizingMessageSource) TestUtils.getPropertyValue(adapter, "source");
assertThat(source).isNotNull();
PriorityBlockingQueue<?> blockingQueue =
@@ -87,6 +90,10 @@ public class InboundChannelAdapterParserTests {
assertThat(TestUtils.getPropertyValue(synchronizer, "preserveTimestamp", Boolean.class)).isTrue();
String remoteFileSeparator = (String) TestUtils.getPropertyValue(synchronizer, "remoteFileSeparator");
assertThat(TestUtils.getPropertyValue(synchronizer, "temporaryFileSuffix", String.class)).isEqualTo(".bar");
assertThat(TestUtils.getPropertyValue(synchronizer, "remoteFileMetadataStore"))
.isSameAs(context.getBean(MetadataStore.class));
assertThat(TestUtils.getPropertyValue(synchronizer, "metadataStorePrefix", String.class))
.isEqualTo("testPrefix");
assertThat(remoteFileSeparator).isNotNull();
assertThat(remoteFileSeparator).isEqualTo(".");
PollableChannel requestChannel = context.getBean("requestChannel", PollableChannel.class);
@@ -103,7 +110,7 @@ public class InboundChannelAdapterParserTests {
@Test
public void testAutoChannel() {
ConfigurableApplicationContext context =
new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context.xml", this.getClass());
new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context.xml", this.getClass());
// Auto-created channel
MessageChannel autoChannel = context.getBean("autoChannel", MessageChannel.class);
SourcePollingChannelAdapter autoChannelAdapter = context.getBean("autoChannel.adapter",
@@ -117,30 +124,36 @@ public class InboundChannelAdapterParserTests {
context.close();
}
@Test(expected = BeanDefinitionStoreException.class)
@Test
//exactly one of 'filename-pattern' or 'filter' is allowed on SFTP inbound adapter
public void testFailWithFilePatternAndFilter() throws Exception {
public void testFailWithFilePatternAndFilter() {
assertThat(!new File("target/bar").exists()).isTrue();
new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context-fail.xml", this.getClass()).close();
assertThatExceptionOfType(BeanDefinitionStoreException.class)
.isThrownBy(() ->
new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context-fail.xml",
getClass()));
}
@Test
public void testLocalDirAutoCreated() throws Exception {
public void testLocalDirAutoCreated() {
assertThat(new File("foo").exists()).isFalse();
ConfigurableApplicationContext context = new ClassPathXmlApplicationContext(
"InboundChannelAdapterParserTests-context.xml", this.getClass());
"InboundChannelAdapterParserTests-context.xml", getClass());
assertThat(new File("foo").exists()).isTrue();
context.close();
}
@Test(expected = BeanCreationException.class)
public void testLocalDirAutoCreateFailed() throws Exception {
new ClassPathXmlApplicationContext("InboundChannelAdapterParserTests-context-fail-autocreate.xml",
this.getClass()).close();
@Test
public void testLocalDirAutoCreateFailed() {
assertThatExceptionOfType(BeanCreationException.class)
.isThrownBy(() ->
new ClassPathXmlApplicationContext(
"InboundChannelAdapterParserTests-context-fail-autocreate.xml",
getClass()));
}
@After
public void cleanUp() throws Exception {
public void cleanUp() {
new File("foo").delete();
}

1
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/dsl/SftpTests.java
View File

@@ -118,6 +118,7 @@ public class SftpTests extends SftpTestSupport {
assertThat(message).isNotNull();
assertThat(message.getPayload()).isInstanceOf(InputStream.class);
assertThat(message.getHeaders().get(FileHeaders.REMOTE_FILE)).isIn(" sftpSource1.txt", "sftpSource2.txt");
assertThat(message.getHeaders().get(FileHeaders.REMOTE_HOST_PORT, String.class)).contains("localhost:");
((InputStream) message.getPayload()).close();
new IntegrationMessageHeaderAccessor(message).getCloseableResource().close();

21
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpInboundRemoteFileSystemSynchronizerTests.java
View File

@@ -36,6 +36,7 @@ import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.integration.file.FileHeaders;
import org.springframework.integration.file.filters.AcceptOnceFileListFilter;
import org.springframework.integration.file.filters.CompositeFileListFilter;
import org.springframework.integration.file.filters.FileListFilter;
@@ -58,6 +59,7 @@ import com.jcraft.jsch.SftpATTRS;
* @author Gunnar Hillert
* @author Gary Russell
* @author Artem Bilan
*
* @since 2.0
*/
public class SftpInboundRemoteFileSystemSynchronizerTests {
@@ -97,7 +99,7 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
store.afterPropertiesSet();
SftpPersistentAcceptOnceFileListFilter persistFilter =
new SftpPersistentAcceptOnceFileListFilter(store, "foo");
List<FileListFilter<LsEntry>> filters = new ArrayList<FileListFilter<LsEntry>>();
List<FileListFilter<LsEntry>> filters = new ArrayList<>();
filters.add(persistFilter);
filters.add(patternFilter);
CompositeFileListFilter<LsEntry> filter = new CompositeFileListFilter<LsEntry>(filters);
@@ -109,27 +111,30 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
ms.setAutoCreateLocalDirectory(true);
ms.setLocalDirectory(localDirectory);
ms.setBeanFactory(mock(BeanFactory.class));
CompositeFileListFilter<File> localFileListFilter = new CompositeFileListFilter<File>();
CompositeFileListFilter<File> localFileListFilter = new CompositeFileListFilter<>();
localFileListFilter.addFilter(new RegexPatternFileListFilter(".*\\.test$"));
AcceptOnceFileListFilter<File> localAcceptOnceFilter = new AcceptOnceFileListFilter<File>();
AcceptOnceFileListFilter<File> localAcceptOnceFilter = new AcceptOnceFileListFilter<>();
localFileListFilter.addFilter(localAcceptOnceFilter);
ms.setLocalFilter(localFileListFilter);
ms.afterPropertiesSet();
ms.start();
Message<File> atestFile = ms.receive();
Message<File> atestFile = ms.receive();
assertThat(atestFile).isNotNull();
assertThat(atestFile.getPayload().getName()).isEqualTo("a.test");
// The test remote files are created with the current timestamp + 1 day.
assertThat(atestFile.getPayload().lastModified()).isGreaterThan(System.currentTimeMillis());
Message<File> btestFile = ms.receive();
assertThat(atestFile.getHeaders())
.containsKeys(FileHeaders.REMOTE_HOST_PORT, FileHeaders.REMOTE_DIRECTORY, FileHeaders.REMOTE_FILE);
Message<File> btestFile = ms.receive();
assertThat(btestFile).isNotNull();
assertThat(btestFile.getPayload().getName()).isEqualTo("b.test");
// The test remote files are created with the current timestamp + 1 day.
assertThat(atestFile.getPayload().lastModified()).isGreaterThan(System.currentTimeMillis());
Message<File> nothing = ms.receive();
Message<File> nothing = ms.receive();
assertThat(nothing).isNull();
// two times because on the third receive (above) the internal queue will be empty, so it will attempt
@@ -143,7 +148,7 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
new File("test/a.test").delete();
new File("test/b.test").delete();
// the remote filter should prevent a re-fetch
nothing = ms.receive();
nothing = ms.receive();
assertThat(nothing).isNull();
ms.stop();
@@ -153,7 +158,7 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
public static class TestSftpSessionFactory extends DefaultSftpSessionFactory {
private final Vector<LsEntry> sftpEntries = new Vector<LsEntry>();
private final Vector<LsEntry> sftpEntries = new Vector<>();
private void init() {
String[] files = new File("remote-test-dir").list();

1
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSourceTests.java
View File

@@ -119,6 +119,7 @@ public class SftpStreamingMessageSourceTests extends SftpTestSupport {
received = (Message<byte[]>) this.data.receive(10000);
assertThat(received).isNotNull();
assertThat(received.getHeaders().get(FileHeaders.REMOTE_FILE_INFO)).isInstanceOf(SftpFileInfo.class);
assertThat(received.getHeaders().get(FileHeaders.REMOTE_HOST_PORT, String.class)).contains("localhost:");
this.adapter.stop();
}