Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-3043: Add FileHeaders.REMOTE_HOST header (#3044)

Browse Source 
* GH-3043: Add FileHeaders.REMOTE_HOST header

Fixes https://github.com/spring-projects/spring-integration/issues/3043

* Populate a `FileHeaders.REMOTE_HOST` from the
`AbstractRemoteFileStreamingMessageSource` and "get"-based commands
in the `AbstractRemoteFileOutboundGateway`
* Extract the value from the a `Session.getHost()` contract
* The `AbstractInboundFileSynchronizingMessageSource` cannot be
addressed with this because the real message is already based on the
locally stored file
* Adjust some affected tests according our code style requirements

* * Add remote file info support into `AbstractInboundFileSynchronizingMessageSource`
* Introduce a `MetadataStore` functionality into the `AbstractInboundFileSynchronizer`
to gather a remote file info an save it in the URI style against local file
* Retrieve such an info in the `AbstractInboundFileSynchronizingMessageSource`
during local file polling
* Introduce `protocol()` contract for the `AbstractInboundFileSynchronizer`
to build a proper URI in the metadata for external readers to distinguish
remote files properly
* Document the feature

* * Fix some typos in Docs

* * Rename property and header constant to the `HOST_PORT` pair
* Fix typos in Docs
* Add  `remote-file-metadata-store` and `metadata-store-prefix` into XSD
of (S)FTP Inbound Channel Adapters
* Add `remoteFileMetadataStore` and `metadataStorePrefix` options
into `RemoteFileInboundChannelAdapterSpec` for Java DSL
This commit is contained in:
Artem Bilan
2019-08-28 08:58:14 -04:00
committed by Gary Russell
parent ff15d5265d
commit a756e6334d
33 changed files with 605 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java
View File

@@ -17,7 +17,8 @@
package org.springframework.integration.file.remote.gateway;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.fail;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
@@ -58,6 +59,7 @@ import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.file.support.FileExistsMode;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageDeliveryException;
import org.springframework.messaging.MessageHandlingException;
import org.springframework.messaging.MessagingException;
import org.springframework.messaging.support.GenericMessage;
@@ -81,11 +83,11 @@ public class RemoteFileOutboundGatewayTests {
public final TemporaryFolder tempFolder = new TemporaryFolder();
@Test(expected = IllegalArgumentException.class)
@Test
public void testBad() {
SessionFactory sessionFactory = mock(SessionFactory.class);
TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway(sessionFactory, "bad", "payload");
gw.afterPropertiesSet();
assertThatIllegalArgumentException()
.isThrownBy(() -> new TestRemoteFileOutboundGateway(sessionFactory, "bad", "payload"));
}
@Test
@@ -93,13 +95,10 @@ public class RemoteFileOutboundGatewayTests {
SessionFactory sessionFactory = mock(SessionFactory.class);
TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway(sessionFactory, "get", "payload");
gw.setFilter(new TestPatternFilter(""));
try {
gw.afterPropertiesSet();
fail("Exception expected");
}
catch (IllegalArgumentException e) {
assertThat(e.getMessage().startsWith("Filters are not supported")).isTrue();
}
assertThatIllegalArgumentException()
.isThrownBy(gw::afterPropertiesSet)
.withMessageStartingWith("Filters are not supported");
}
@Test
@@ -107,13 +106,10 @@ public class RemoteFileOutboundGatewayTests {
SessionFactory sessionFactory = mock(SessionFactory.class);
TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway(sessionFactory, "rm", "payload");
gw.setFilter(new TestPatternFilter(""));
try {
gw.afterPropertiesSet();
fail("Exception expected");
}
catch (IllegalArgumentException e) {
assertThat(e.getMessage().startsWith("Filters are not supported")).isTrue();
}
assertThatIllegalArgumentException()
.isThrownBy(gw::afterPropertiesSet)
.withMessageStartingWith("Filters are not supported");
}
@Test
@@ -139,11 +135,6 @@ public class RemoteFileOutboundGatewayTests {
testMGetWildGuts("f1", "f2");
}
/**
* Test a wildcard mget where the full path is returned for each file
*
* @throws Exception
*/
@Test
public void testMGetWildFullPath() {
testMGetWildGuts("testremote/f1", "testremote/f2");
@@ -547,7 +538,7 @@ public class RemoteFileOutboundGatewayTests {
}
@Test
public void testGet() throws Exception {
public void testGet() {
SessionFactory sessionFactory = mock(SessionFactory.class);
TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway(sessionFactory, "get", "payload");
gw.setLocalDirectory(new File(this.tmpDir));
@@ -609,22 +600,16 @@ public class RemoteFileOutboundGatewayTests {
// default (null)
MessageBuilder<File> out;
try {
out = (MessageBuilder<File>) gw.handleRequestMessage(new GenericMessage<>("f1"));
fail("Exception expected");
}
catch (MessageHandlingException e) {
assertThat(e.getMessage()).contains("already exists");
}
assertThatExceptionOfType(MessageHandlingException.class)
.isThrownBy(() -> gw.handleRequestMessage(new GenericMessage<>("f1")))
.withMessageContaining("already exists");
gw.setFileExistsMode(FileExistsMode.FAIL);
try {
out = (MessageBuilder<File>) gw.handleRequestMessage(new GenericMessage<>("f1"));
fail("Exception expected");
}
catch (MessageHandlingException e) {
assertThat(e.getMessage()).contains("already exists");
}
assertThatExceptionOfType(MessageHandlingException.class)
.isThrownBy(() -> gw.handleRequestMessage(new GenericMessage<>("f1")))
.withMessageContaining("already exists");
gw.setFileExistsMode(FileExistsMode.IGNORE);
out = (MessageBuilder<File>) gw.handleRequestMessage(new GenericMessage<>("f1"));
@@ -652,7 +637,8 @@ public class RemoteFileOutboundGatewayTests {
@Test
public void testGetTempFileDelete() {
SessionFactory sessionFactory = mock(SessionFactory.class);
@SuppressWarnings("unchecked")
SessionFactory<TestLsEntry> sessionFactory = mock(SessionFactory.class);
TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway(sessionFactory, "get", "payload");
gw.setLocalDirectory(new File(this.tmpDir));
gw.afterPropertiesSet();
@@ -672,18 +658,15 @@ public class RemoteFileOutboundGatewayTests {
}
});
try {
gw.handleRequestMessage(new GenericMessage<>("f1"));
fail("Expected exception");
}
catch (MessagingException e) {
assertThat(e.getCause()).isInstanceOf(RuntimeException.class);
assertThat(e.getCause().getMessage()).isEqualTo("test remove .writing");
@SuppressWarnings("unchecked")
RemoteFileTemplate template = new RemoteFileTemplate(sessionFactory);
File outFile = new File(this.tmpDir + "/f1" + template.getTemporaryFileSuffix());
assertThat(outFile.exists()).isFalse();
}
assertThatExceptionOfType(MessagingException.class)
.isThrownBy(() -> gw.handleRequestMessage(new GenericMessage<>("f1")))
.withCauseInstanceOf(RuntimeException.class)
.withMessageContaining("test remove .writing");
RemoteFileTemplate<?> template = new RemoteFileTemplate<>(sessionFactory);
File outFile = new File(this.tmpDir + "/f1" + template.getTemporaryFileSuffix());
assertThat(outFile.exists()).isFalse();
}
@@ -743,8 +726,7 @@ public class RemoteFileOutboundGatewayTests {
}
@Override
public void read(String source, OutputStream outputStream)
throws IOException {
public void read(String source, OutputStream outputStream) throws IOException {
outputStream.write("testfile".getBytes());
}
@@ -844,13 +826,10 @@ public class RemoteFileOutboundGatewayTests {
verify(session).rename("foo/bar.txt.writing", "foo/bar.txt");
gw.setFileExistsMode(FileExistsMode.FAIL);
try {
path = (String) gw.handleRequestMessage(requestMessage);
fail("Exception expected");
}
catch (Exception e) {
assertThat(e.getMessage()).contains("The destination file already exists");
}
assertThatExceptionOfType(MessageDeliveryException.class)
.isThrownBy(() -> gw.handleRequestMessage(requestMessage))
.withMessageContaining("The destination file already exists");
gw.setFileExistsMode(FileExistsMode.REPLACE);
path = (String) gw.handleRequestMessage(requestMessage);
@@ -876,10 +855,9 @@ public class RemoteFileOutboundGatewayTests {
}
@Test
@SuppressWarnings("unchecked")
public void testMput() throws Exception {
@SuppressWarnings("unchecked")
SessionFactory<TestLsEntry> sessionFactory = mock(SessionFactory.class);
@SuppressWarnings("unchecked")
Session<TestLsEntry> session = mock(Session.class);
RemoteFileTemplate<TestLsEntry> template = new RemoteFileTemplate<>(sessionFactory);
template.setRemoteDirectoryExpression(new LiteralExpression("foo/"));
@@ -897,7 +875,6 @@ public class RemoteFileOutboundGatewayTests {
tempFolder.newFile("qux.txt");
Message<File> requestMessage = MessageBuilder.withPayload(tempFolder.getRoot())
.build();
@SuppressWarnings("unchecked")
List<String> out = (List<String>) gw.handleRequestMessage(requestMessage);
assertThat(out.size()).isEqualTo(2);
assertThat(out.get(0)).isNotEqualTo(out.get(1));
@@ -906,10 +883,9 @@ public class RemoteFileOutboundGatewayTests {
}
@Test
@SuppressWarnings("unchecked")
public void testMputRecursive() throws Exception {
@SuppressWarnings("unchecked")
SessionFactory<TestLsEntry> sessionFactory = mock(SessionFactory.class);
@SuppressWarnings("unchecked")
Session<TestLsEntry> session = mock(Session.class);
RemoteFileTemplate<TestLsEntry> template = new RemoteFileTemplate<>(sessionFactory);
template.setRemoteDirectoryExpression(new LiteralExpression("foo/"));
@@ -931,7 +907,6 @@ public class RemoteFileOutboundGatewayTests {
Message<File> requestMessage = MessageBuilder.withPayload(tempFolder.getRoot())
.build();
@SuppressWarnings("unchecked")
List<String> out = (List<String>) gw.handleRequestMessage(requestMessage);
assertThat(out.size()).isEqualTo(3);
assertThat(out.get(0)).isNotEqualTo(out.get(1));
@@ -941,10 +916,9 @@ public class RemoteFileOutboundGatewayTests {
}
@Test
@SuppressWarnings("unchecked")
public void testMputCollection() throws Exception {
@SuppressWarnings("unchecked")
SessionFactory<TestLsEntry> sessionFactory = mock(SessionFactory.class);
@SuppressWarnings("unchecked")
Session<TestLsEntry> session = mock(Session.class);
RemoteFileTemplate<TestLsEntry> template = new RemoteFileTemplate<>(sessionFactory);
template.setRemoteDirectoryExpression(new LiteralExpression("foo/"));
@@ -963,7 +937,6 @@ public class RemoteFileOutboundGatewayTests {
files.add(tempFolder.newFile("buz.txt"));
Message<List<File>> requestMessage = MessageBuilder.withPayload(files)
.build();
@SuppressWarnings("unchecked")
List<String> out = (List<String>) gw.handleRequestMessage(requestMessage);
assertThat(out.size()).isEqualTo(2);
assertThat(out.get(0)).isNotEqualTo(out.get(1));
@@ -1022,7 +995,7 @@ public class RemoteFileOutboundGatewayTests {
@Override
public boolean isOpen() {
return open;
return this.open;
}
@Override
@@ -1050,6 +1023,11 @@ public class RemoteFileOutboundGatewayTests {
return null;
}
@Override
public String getHostPort() {
return null;
}
}

65
spring-integration-file/src/test/java/org/springframework/integration/file/remote/session/CachingSessionFactoryTests.java
View File

@@ -17,13 +17,12 @@
package org.springframework.integration.file.remote.session;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.fail;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -31,13 +30,15 @@ import org.junit.Test;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.expression.common.LiteralExpression;
import org.springframework.integration.file.remote.InputStreamCallback;
import org.springframework.integration.file.remote.RemoteFileTemplate;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.messaging.MessagingException;
import org.springframework.messaging.support.GenericMessage;
/**
* @author Gary Russell
* @author Artem Bilan
*
* @since 3.0
*
*/
@@ -46,7 +47,7 @@ public class CachingSessionFactoryTests {
@Test
public void testCacheAndReset() {
TestSessionFactory factory = new TestSessionFactory();
CachingSessionFactory<String> cache = new CachingSessionFactory<String>(factory);
CachingSessionFactory<String> cache = new CachingSessionFactory<>(factory);
cache.setTestSession(true);
Session<String> sess1 = cache.getSession();
assertThat(TestUtils.getPropertyValue(sess1, "targetSession.id")).isEqualTo("session:1");
@@ -83,25 +84,24 @@ public class CachingSessionFactoryTests {
when(factory.getSession()).thenReturn(session);
when(session.readRaw("foo")).thenReturn(new ByteArrayInputStream("".getBytes()));
when(session.finalizeRaw()).thenReturn(true);
CachingSessionFactory<Object> ccf = new CachingSessionFactory<Object>(factory);
RemoteFileTemplate<Object> template = new RemoteFileTemplate<Object>(ccf);
CachingSessionFactory<Object> ccf = new CachingSessionFactory<>(factory);
RemoteFileTemplate<Object> template = new RemoteFileTemplate<>(ccf);
template.setFileNameExpression(new LiteralExpression("foo"));
template.setBeanFactory(mock(BeanFactory.class));
template.afterPropertiesSet();
try {
template.get(new GenericMessage<String>("foo"), (InputStreamCallback) stream -> {
throw new RuntimeException("bar");
});
fail("Expected exception");
}
catch (Exception e) {
assertThat(e.getCause()).isInstanceOf(RuntimeException.class);
assertThat(e.getCause().getMessage()).isEqualTo("bar");
}
assertThatExceptionOfType(MessagingException.class)
.isThrownBy(() ->
template.get(new GenericMessage<>("foo"),
stream -> {
throw new RuntimeException("bar");
}))
.withCauseInstanceOf(RuntimeException.class)
.withMessageContaining("bar");
verify(session).close();
}
private class TestSessionFactory implements SessionFactory<String> {
private static class TestSessionFactory implements SessionFactory<String> {
private int n;
@@ -112,7 +112,7 @@ public class CachingSessionFactoryTests {
}
private class TestSession implements Session<String> {
private static class TestSession implements Session<String> {
@SuppressWarnings("unused")
private final String id;
@@ -127,39 +127,39 @@ public class CachingSessionFactoryTests {
}
@Override
public boolean remove(String path) throws IOException {
public boolean remove(String path) {
return false;
}
@Override
public String[] list(String path) throws IOException {
public String[] list(String path) {
return null;
}
@Override
public void read(String source, OutputStream outputStream) throws IOException {
public void read(String source, OutputStream outputStream) {
}
@Override
public void write(InputStream inputStream, String destination) throws IOException {
public void write(InputStream inputStream, String destination) {
}
@Override
public void append(InputStream inputStream, String destination) throws IOException {
public void append(InputStream inputStream, String destination) {
}
@Override
public boolean mkdir(String directory) throws IOException {
public boolean mkdir(String directory) {
return false;
}
@Override
public boolean rmdir(String directory) throws IOException {
public boolean rmdir(String directory) {
return false;
}
@Override
public void rename(String pathFrom, String pathTo) throws IOException {
public void rename(String pathFrom, String pathTo) {
}
@Override
@@ -173,22 +173,22 @@ public class CachingSessionFactoryTests {
}
@Override
public boolean exists(String path) throws IOException {
public boolean exists(String path) {
return false;
}
@Override
public String[] listNames(String path) throws IOException {
public String[] listNames(String path) {
return null;
}
@Override
public InputStream readRaw(String source) throws IOException {
public InputStream readRaw(String source) {
return null;
}
@Override
public boolean finalizeRaw() throws IOException {
public boolean finalizeRaw() {
return false;
}
@@ -197,6 +197,11 @@ public class CachingSessionFactoryTests {
return null;
}
@Override
public String getHostPort() {
return null;
}
@Override
public boolean test() {
this.testCalled = true;

40
spring-integration-file/src/test/java/org/springframework/integration/file/remote/synchronizer/AbstractRemoteFileSynchronizerTests.java
View File

@@ -17,7 +17,7 @@
package org.springframework.integration.file.remote.synchronizer;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.fail;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.mockito.Mockito.mock;
import java.io.File;
@@ -72,6 +72,11 @@ public class AbstractRemoteFileSynchronizerTests {
return 0;
}
@Override
protected String protocol() {
return "file";
}
@Override
protected boolean copyFileToLocalDirectory(String remoteDirectoryPath, String remoteFile,
File localDirectory, Session<String> session) throws IOException {
@@ -83,19 +88,14 @@ public class AbstractRemoteFileSynchronizerTests {
}
};
sync.setFilter(new AcceptOnceFileListFilter<String>());
sync.setFilter(new AcceptOnceFileListFilter<>());
sync.setRemoteDirectory("foo");
try {
sync.synchronizeToLocalDirectory(mock(File.class));
assertThat(count.get()).isEqualTo(1);
fail("Expected exception");
}
catch (MessagingException e) {
assertThat(e.getCause()).isInstanceOf(MessagingException.class);
assertThat(e.getCause().getCause()).isInstanceOf(IOException.class);
assertThat(e.getCause().getCause().getMessage()).isEqualTo("fail");
}
assertThatExceptionOfType(MessagingException.class)
.isThrownBy(() -> sync.synchronizeToLocalDirectory(mock(File.class)))
.withRootCauseInstanceOf(IOException.class)
.withMessageContaining("fail");
sync.synchronizeToLocalDirectory(mock(File.class));
assertThat(count.get()).isEqualTo(3);
sync.close();
@@ -261,6 +261,11 @@ public class AbstractRemoteFileSynchronizerTests {
return 0;
}
@Override
protected String protocol() {
return "file";
}
@Override
protected boolean copyFileToLocalDirectory(String remoteDirectoryPath, String remoteFile,
File localDirectory, Session<String> session) {
@@ -269,13 +274,13 @@ public class AbstractRemoteFileSynchronizerTests {
}
};
sync.setFilter(new AcceptOnceFileListFilter<String>());
sync.setFilter(new AcceptOnceFileListFilter<>());
sync.setRemoteDirectory("foo");
sync.setBeanFactory(mock(BeanFactory.class));
return sync;
}
private class StringSessionFactory implements SessionFactory<String> {
private static class StringSessionFactory implements SessionFactory<String> {
@Override
public Session<String> getSession() {
@@ -284,7 +289,7 @@ public class AbstractRemoteFileSynchronizerTests {
}
private class StringSession implements Session<String> {
private static class StringSession implements Session<String> {
StringSession() {
super();
@@ -360,6 +365,11 @@ public class AbstractRemoteFileSynchronizerTests {
return null;
}
@Override
public String getHostPort() {
return null;
}
}
}