GH-3043: Add FileHeaders.REMOTE_HOST header (#3044)

* GH-3043: Add FileHeaders.REMOTE_HOST header

Fixes https://github.com/spring-projects/spring-integration/issues/3043

* Populate a `FileHeaders.REMOTE_HOST` from the
`AbstractRemoteFileStreamingMessageSource` and "get"-based commands
in the `AbstractRemoteFileOutboundGateway`
* Extract the value from the a `Session.getHost()` contract
* The `AbstractInboundFileSynchronizingMessageSource` cannot be
addressed with this because the real message is already based on the
locally stored file
* Adjust some affected tests according our code style requirements

* * Add remote file info support into `AbstractInboundFileSynchronizingMessageSource`
* Introduce a `MetadataStore` functionality into the `AbstractInboundFileSynchronizer`
to gather a remote file info an save it in the URI style against local file
* Retrieve such an info in the `AbstractInboundFileSynchronizingMessageSource`
during local file polling
* Introduce `protocol()` contract for the `AbstractInboundFileSynchronizer`
to build a proper URI in the metadata for external readers to distinguish
remote files properly
* Document the feature

* * Fix some typos in Docs

* * Rename property and header constant to the `HOST_PORT` pair
* Fix typos in Docs
* Add  `remote-file-metadata-store` and `metadata-store-prefix` into XSD
of (S)FTP Inbound Channel Adapters
* Add `remoteFileMetadataStore` and `metadataStorePrefix` options
into `RemoteFileInboundChannelAdapterSpec` for Java DSL
This commit is contained in:
Artem Bilan
2019-08-28 08:58:14 -04:00
committed by Gary Russell
parent ff15d5265d
commit a756e6334d
33 changed files with 605 additions and 297 deletions

View File

@@ -32,6 +32,7 @@ import org.springframework.integration.metadata.SimpleMetadataStore;
* @author Mark Fisher
* @author Artem Bilan
* @author Gary Russell
*
* @since 2.0
*/
public class FtpInboundFileSynchronizer extends AbstractInboundFileSynchronizer<FTPFile> {
@@ -62,4 +63,9 @@ public class FtpInboundFileSynchronizer extends AbstractInboundFileSynchronizer<
return file.getTimestamp().getTimeInMillis();
}
@Override
protected String protocol() {
return "ftp";
}
}

View File

@@ -38,13 +38,14 @@ import org.springframework.util.ObjectUtils;
* @author Oleg Zhurakousky
* @author Gary Russell
* @author Artem Bilan
*
* @since 2.0
*/
public class FtpSession implements Session<FTPFile> {
private static final String SERVER_REPLIED_WITH = "'. Server replied with: ";
private static final Log LOGGER = LogFactory.getLog(FtpSession.class);
private final Log logger = LogFactory.getLog(this.getClass());
private static final String SERVER_REPLIED_WITH = "'. Server replied with: ";
private final FTPClient client;
@@ -86,7 +87,9 @@ public class FtpSession implements Session<FTPFile> {
throw new IOException("Failed to copy '" + path +
SERVER_REPLIED_WITH + this.client.getReplyString());
}
this.logger.info("File has been successfully transferred from: " + path);
if (LOGGER.isInfoEnabled()) {
LOGGER.info("File has been successfully transferred from: " + path);
}
}
@Override
@@ -109,8 +112,8 @@ public class FtpSession implements Session<FTPFile> {
}
if (this.client.completePendingCommand()) {
int replyCode = this.client.getReplyCode();
if (this.logger.isDebugEnabled()) {
this.logger.debug(this + " finalizeRaw - reply code: " + replyCode);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(this + " finalizeRaw - reply code: " + replyCode);
}
return FTPReply.isPositiveCompletion(replyCode);
}
@@ -126,8 +129,8 @@ public class FtpSession implements Session<FTPFile> {
throw new IOException("Failed to write to '" + path
+ SERVER_REPLIED_WITH + this.client.getReplyString());
}
if (this.logger.isInfoEnabled()) {
this.logger.info("File has been successfully transferred to: " + path);
if (LOGGER.isInfoEnabled()) {
LOGGER.info("File has been successfully transferred to: " + path);
}
}
@@ -140,8 +143,8 @@ public class FtpSession implements Session<FTPFile> {
throw new IOException("Failed to append to '" + path
+ SERVER_REPLIED_WITH + this.client.getReplyString());
}
if (this.logger.isInfoEnabled()) {
this.logger.info("File has been successfully appended to: " + path);
if (LOGGER.isInfoEnabled()) {
LOGGER.info("File has been successfully appended to: " + path);
}
}
@@ -150,17 +153,15 @@ public class FtpSession implements Session<FTPFile> {
try {
if (this.readingRaw.get()) {
if (!finalizeRaw()) {
if (this.logger.isWarnEnabled()) {
this.logger.warn("Finalize on readRaw() returned false for " + this);
if (LOGGER.isWarnEnabled()) {
LOGGER.warn("Finalize on readRaw() returned false for " + this);
}
}
}
this.client.disconnect();
}
catch (Exception e) {
if (this.logger.isWarnEnabled()) {
this.logger.warn("failed to disconnect FTPClient", e);
}
LOGGER.warn("failed to disconnect FTPClient", e);
}
}
@@ -183,8 +184,8 @@ public class FtpSession implements Session<FTPFile> {
throw new IOException("Failed to rename '" + pathFrom +
"' to " + pathTo + SERVER_REPLIED_WITH + this.client.getReplyString());
}
if (this.logger.isInfoEnabled()) {
this.logger.info("File has been successfully renamed from: " + pathFrom + " to " + pathTo);
if (LOGGER.isInfoEnabled()) {
LOGGER.info("File has been successfully renamed from: " + pathFrom + " to " + pathTo);
}
}
@@ -228,6 +229,10 @@ public class FtpSession implements Session<FTPFile> {
return this.client;
}
@Override
public String getHostPort() {
return this.client.getRemoteAddress().getHostName() + ':' + this.client.getRemotePort();
}
@Override
public boolean test() {

View File

@@ -177,6 +177,29 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="remote-file-metadata-store" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type
type="org.springframework.integration.metadata.MetadataStore" />
</tool:annotation>
</xsd:appinfo>
<xsd:documentation>
Reference to a MetadataStore for saving remote files information between
synchronization and polling.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="metadata-store-prefix" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Specify a prefix for metadata store to distinguish keys from another places
where the same shared store is used.
By default, the remote a component name is used.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attributeGroup ref="tempSuffixGroup" />
</xsd:extension>
</xsd:complexContent>

View File

@@ -30,12 +30,16 @@
temporary-file-suffix=".foo"
max-fetch-size="42"
local-filter="acceptAllFilter"
remote-directory-expression="'foo/bar'">
remote-directory-expression="'foo/bar'"
remote-file-metadata-store="metadataStore"
metadata-store-prefix="testPrefix">
<int:poller fixed-rate="1000">
<int:transactional synchronization-factory="syncFactory"/>
</int:poller>
</int-ftp:inbound-channel-adapter>
<bean id="metadataStore" class="org.springframework.integration.metadata.SimpleMetadataStore"/>
<bean id="dirScanner" class="org.springframework.integration.file.HeadDirectoryScanner">
<constructor-arg value="1" />
</bean>

View File

@@ -48,6 +48,7 @@ import org.springframework.integration.ftp.inbound.FtpInboundFileSynchronizer;
import org.springframework.integration.ftp.inbound.FtpInboundFileSynchronizingMessageSource;
import org.springframework.integration.ftp.session.DefaultFtpSessionFactory;
import org.springframework.integration.ftp.session.FtpSession;
import org.springframework.integration.metadata.MetadataStore;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.messaging.MessageChannel;
import org.springframework.test.annotation.DirtiesContext;
@@ -87,6 +88,9 @@ public class FtpInboundChannelAdapterParserTests {
@Autowired
private DirectoryScanner dirScanner;
@Autowired
private MetadataStore metadataStore;
@Test
public void testFtpInboundChannelAdapterComplete() throws Exception {
assertThat(TestUtils.getPropertyValue(ftpInbound, "autoStartup", Boolean.class)).isFalse();
@@ -109,6 +113,9 @@ public class FtpInboundChannelAdapterParserTests {
assertThat(TestUtils.getPropertyValue(fisync, "localFilenameGeneratorExpression")).isNotNull();
assertThat(TestUtils.getPropertyValue(fisync, "preserveTimestamp", Boolean.class)).isTrue();
assertThat(TestUtils.getPropertyValue(fisync, "temporaryFileSuffix", String.class)).isEqualTo(".foo");
assertThat(TestUtils.getPropertyValue(fisync, "remoteFileMetadataStore", MetadataStore.class))
.isSameAs(this.metadataStore);
assertThat(TestUtils.getPropertyValue(fisync, "metadataStorePrefix", String.class)).isEqualTo("testPrefix");
String remoteFileSeparator = (String) TestUtils.getPropertyValue(fisync, "remoteFileSeparator");
assertThat(remoteFileSeparator).isNotNull();
assertThat(remoteFileSeparator).isEqualTo("");
@@ -123,11 +130,11 @@ public class FtpInboundChannelAdapterParserTests {
assertThat(filtersIterator.next()).isInstanceOf(FtpPersistentAcceptOnceFileListFilter.class);
Object sessionFactory = TestUtils.getPropertyValue(fisync, "remoteFileTemplate.sessionFactory");
assertThat(DefaultFtpSessionFactory.class.isAssignableFrom(sessionFactory.getClass())).isTrue();
assertThat(sessionFactory).isInstanceOf(DefaultFtpSessionFactory.class);
FileListFilter<?> acceptAllFilter = context.getBean("acceptAllFilter", FileListFilter.class);
assertThat(TestUtils.getPropertyValue(inbound, "fileSource.scanner.filter.fileFilters", Collection.class)
.contains(acceptAllFilter)).isTrue();
final AtomicReference<Method> genMethod = new AtomicReference<Method>();
final AtomicReference<Method> genMethod = new AtomicReference<>();
ReflectionUtils.doWithMethods(AbstractInboundFileSynchronizer.class, method -> {
method.setAccessible(true);
genMethod.set(method);
@@ -137,10 +144,10 @@ public class FtpInboundChannelAdapterParserTests {
}
@Test
public void cachingSessionFactory() throws Exception {
public void cachingSessionFactory() {
Object sessionFactory = TestUtils.getPropertyValue(simpleAdapterWithCachedSessions,
"source.synchronizer.remoteFileTemplate.sessionFactory");
assertThat(sessionFactory.getClass()).isEqualTo(CachingSessionFactory.class);
assertThat(sessionFactory).isInstanceOf(CachingSessionFactory.class);
FtpInboundFileSynchronizer fisync =
TestUtils.getPropertyValue(simpleAdapterWithCachedSessions, "source.synchronizer",
FtpInboundFileSynchronizer.class);
@@ -161,7 +168,7 @@ public class FtpInboundChannelAdapterParserTests {
public static class TestSessionFactoryBean implements FactoryBean<DefaultFtpSessionFactory> {
@Override
public DefaultFtpSessionFactory getObject() throws Exception {
public DefaultFtpSessionFactory getObject() {
DefaultFtpSessionFactory factory = mock(DefaultFtpSessionFactory.class);
FtpSession session = mock(FtpSession.class);
when(factory.getSession()).thenReturn(session);

View File

@@ -98,6 +98,8 @@ public class FtpTests extends FtpTestSupport {
IntegrationFlowRegistration registration = this.flowContext.registration(flow).register();
Message<?> message = out.receive(10_000);
assertThat(message).isNotNull();
assertThat(message.getHeaders())
.containsKeys(FileHeaders.REMOTE_HOST_PORT, FileHeaders.REMOTE_DIRECTORY, FileHeaders.REMOTE_FILE);
Object payload = message.getPayload();
assertThat(payload).isInstanceOf(File.class);
File file = (File) payload;
@@ -153,6 +155,7 @@ public class FtpTests extends FtpTestSupport {
assertThat(message).isNotNull();
assertThat(message.getPayload()).isInstanceOf(InputStream.class);
assertThat(message.getHeaders().get(FileHeaders.REMOTE_FILE)).isIn(" ftpSource1.txt", "ftpSource2.txt");
assertThat(message.getHeaders().get(FileHeaders.REMOTE_HOST_PORT, String.class)).contains("localhost:");
new IntegrationMessageHeaderAccessor(message).getCloseableResource().close();
message = out.receive(10_000);

View File

@@ -25,10 +25,12 @@ import static org.mockito.Mockito.when;
import java.io.File;
import java.io.OutputStream;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
@@ -51,7 +53,9 @@ import org.springframework.integration.file.filters.RegexPatternFileListFilter;
import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter;
import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter;
import org.springframework.integration.ftp.session.AbstractFtpSessionFactory;
import org.springframework.integration.metadata.MetadataStore;
import org.springframework.integration.metadata.PropertiesPersistingMetadataStore;
import org.springframework.integration.metadata.SimpleMetadataStore;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.messaging.Message;
@@ -60,6 +64,7 @@ import org.springframework.messaging.Message;
* @author Gunnar Hillert
* @author Gary Russell
* @author Artem Bilan
*
* @since 2.0
*/
public class FtpInboundRemoteFileSystemSynchronizerTests {
@@ -76,7 +81,7 @@ public class FtpInboundRemoteFileSystemSynchronizerTests {
public void testCopyFileToLocalDir() throws Exception {
File localDirectory = new File("test");
assertThat(localDirectory.exists()).isFalse();
MetadataStore remoteFileMetadataStore = new SimpleMetadataStore();
TestFtpSessionFactory ftpSessionFactory = new TestFtpSessionFactory();
ftpSessionFactory.setUsername("kermit");
ftpSessionFactory.setPassword("frog");
@@ -85,16 +90,18 @@ public class FtpInboundRemoteFileSystemSynchronizerTests {
synchronizer.setDeleteRemoteFiles(true);
synchronizer.setPreserveTimestamp(true);
synchronizer.setRemoteDirectory("remote-test-dir");
synchronizer.setRemoteFileMetadataStore(remoteFileMetadataStore);
synchronizer.setMetadataStorePrefix("ftpPollingTest:");
FtpRegexPatternFileListFilter patternFilter = new FtpRegexPatternFileListFilter(".*\\.test$");
PropertiesPersistingMetadataStore store = spy(new PropertiesPersistingMetadataStore());
store.setBaseDirectory("test");
store.afterPropertiesSet();
FtpPersistentAcceptOnceFileListFilter persistFilter =
new FtpPersistentAcceptOnceFileListFilter(store, "foo");
List<FileListFilter<FTPFile>> filters = new ArrayList<FileListFilter<FTPFile>>();
List<FileListFilter<FTPFile>> filters = new ArrayList<>();
filters.add(persistFilter);
filters.add(patternFilter);
CompositeFileListFilter<FTPFile> filter = new CompositeFileListFilter<FTPFile>(filters);
CompositeFileListFilter<FTPFile> filter = new CompositeFileListFilter<>(filters);
synchronizer.setFilter(filter);
ExpressionParser expressionParser = new SpelExpressionParser(new SpelParserConfiguration(true, true));
@@ -108,9 +115,9 @@ public class FtpInboundRemoteFileSystemSynchronizerTests {
ms.setAutoCreateLocalDirectory(true);
ms.setLocalDirectory(localDirectory);
ms.setBeanFactory(mock(BeanFactory.class));
CompositeFileListFilter<File> localFileListFilter = new CompositeFileListFilter<File>();
CompositeFileListFilter<File> localFileListFilter = new CompositeFileListFilter<>();
localFileListFilter.addFilter(new RegexPatternFileListFilter(".*\\.TEST\\.a$"));
AcceptOnceFileListFilter<File> localAcceptOnceFilter = new AcceptOnceFileListFilter<File>();
AcceptOnceFileListFilter<File> localAcceptOnceFilter = new AcceptOnceFileListFilter<>();
localFileListFilter.addFilter(localAcceptOnceFilter);
RecursiveDirectoryScanner scanner = new RecursiveDirectoryScanner();
ms.setScanner(scanner);
@@ -143,8 +150,12 @@ public class FtpInboundRemoteFileSystemSynchronizerTests {
TestUtils.getPropertyValue(localAcceptOnceFilter, "seenSet", Collection.class).clear();
new File("test/subdir/A.TEST.a").delete();
new File("test/subdir/B.TEST.a").delete();
File aFile = new File("test/subdir/A.TEST.a");
aFile.delete();
synchronizer.removeRemoteFileMetadata(aFile);
File bFile = new File("test/subdir/B.TEST.a");
bFile.delete();
synchronizer.removeRemoteFileMetadata(bFile);
// the remote filter should prevent a re-fetch
nothing = ms.receive();
assertThat(nothing).isNull();
@@ -152,11 +163,14 @@ public class FtpInboundRemoteFileSystemSynchronizerTests {
ms.stop();
verify(synchronizer).close();
verify(store).close();
Map<?, ?> metadata = TestUtils.getPropertyValue(remoteFileMetadataStore, "metadata", Map.class);
assertThat(metadata).isEmpty();
}
@Test
public void testSyncRemoteFileOnlyOnceByDefault() throws Exception {
public void testSyncRemoteFileOnlyOnceByDefault() {
File localDirectory = new File("test");
localDirectory.mkdir();
@@ -204,7 +218,7 @@ public class FtpInboundRemoteFileSystemSynchronizerTests {
public static class TestFtpSessionFactory extends AbstractFtpSessionFactory<FTPClient> {
private final Collection<FTPFile> ftpFiles = new ArrayList<FTPFile>();
private final Collection<FTPFile> ftpFiles = new ArrayList<>();
private void init() {
String[] files = new File("remote-test-dir").list();
@@ -237,8 +251,10 @@ public class FtpInboundRemoteFileSystemSynchronizerTests {
Mockito.any(OutputStream.class))).thenReturn(true);
}
when(ftpClient.listFiles("remote-test-dir"))
.thenReturn(ftpFiles.toArray(new FTPFile[ftpFiles.size()]));
.thenReturn(ftpFiles.toArray(new FTPFile[0]));
when(ftpClient.deleteFile(Mockito.anyString())).thenReturn(true);
when(ftpClient.getRemoteAddress()).thenReturn(InetAddress.getByName("localhost"));
when(ftpClient.getRemotePort()).thenReturn(-1);
return ftpClient;
}
catch (Exception e) {

View File

@@ -123,6 +123,7 @@ public class FtpStreamingMessageSourceTests extends FtpTestSupport {
received = (Message<byte[]>) this.data.receive(10000);
assertThat(received).isNotNull();
assertThat(received.getHeaders().get(FileHeaders.REMOTE_FILE_INFO)).isInstanceOf(FtpFileInfo.class);
assertThat(received.getHeaders().get(FileHeaders.REMOTE_HOST_PORT, String.class)).contains("localhost:");
assertThat(TestUtils.getPropertyValue(source, "toBeReceived", BlockingQueue.class)).hasSize(1);
assertThat(this.metadataMap).hasSize(1);
this.adapter.stop();

View File

@@ -27,6 +27,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
@@ -88,12 +89,12 @@ public class FtpOutboundTests {
file.delete();
}
assertThat(file.exists()).isFalse();
FileTransferringMessageHandler<FTPFile> handler = new FileTransferringMessageHandler<FTPFile>(sessionFactory);
FileTransferringMessageHandler<FTPFile> handler = new FileTransferringMessageHandler<>(sessionFactory);
handler.setRemoteDirectoryExpression(new LiteralExpression("remote-target-dir"));
handler.setFileNameGenerator(message -> "handlerContent.test");
handler.setBeanFactory(mock(BeanFactory.class));
handler.afterPropertiesSet();
handler.handleMessage(new GenericMessage<String>("String data"));
handler.handleMessage(new GenericMessage<>("String data"));
assertThat(file.exists()).isTrue();
byte[] inFile = FileCopyUtils.copyToByteArray(file);
assertThat(new String(inFile)).isEqualTo("String data");
@@ -107,12 +108,12 @@ public class FtpOutboundTests {
file.delete();
}
assertThat(file.exists()).isFalse();
FileTransferringMessageHandler<FTPFile> handler = new FileTransferringMessageHandler<FTPFile>(sessionFactory);
FileTransferringMessageHandler<FTPFile> handler = new FileTransferringMessageHandler<>(sessionFactory);
handler.setRemoteDirectoryExpression(new LiteralExpression("remote-target-dir"));
handler.setFileNameGenerator(message -> "handlerContent.test");
handler.setBeanFactory(mock(BeanFactory.class));
handler.afterPropertiesSet();
handler.handleMessage(new GenericMessage<byte[]>("byte[] data".getBytes()));
handler.handleMessage(new GenericMessage<>("byte[] data".getBytes()));
assertThat(file.exists()).isTrue();
byte[] inFile = FileCopyUtils.copyToByteArray(file);
assertThat(new String(inFile)).isEqualTo("byte[] data");
@@ -124,7 +125,7 @@ public class FtpOutboundTests {
File targetDir = new File("remote-target-dir");
assertThat(targetDir.exists()).as("target directory does not exist: " + targetDir.getName()).isTrue();
FileTransferringMessageHandler<FTPFile> handler = new FileTransferringMessageHandler<FTPFile>(sessionFactory);
FileTransferringMessageHandler<FTPFile> handler = new FileTransferringMessageHandler<>(sessionFactory);
handler.setRemoteDirectoryExpression(new LiteralExpression(targetDir.getName()));
handler.setFileNameGenerator(message -> ((File) message.getPayload()).getName() + ".test");
handler.setBeanFactory(mock(BeanFactory.class));
@@ -141,7 +142,7 @@ public class FtpOutboundTests {
}
@Test
public void testHandleMissingFileMessage() throws Exception {
public void testHandleMissingFileMessage() {
File targetDir = new File("remote-target-dir");
assertThat(targetDir.exists()).as("target directory does not exist: " + targetDir.getName()).isTrue();
@@ -191,7 +192,7 @@ public class FtpOutboundTests {
}
@Test //INT-2275
public void testFtpOutboundGatewayInsideChain() throws Exception {
public void testFtpOutboundGatewayInsideChain() {
ConfigurableApplicationContext context = new ClassPathXmlApplicationContext(
"FtpOutboundInsideChainTests-context.xml", getClass());
@@ -248,7 +249,9 @@ public class FtpOutboundTests {
any(OutputStream.class))).thenReturn(true);
}
when(ftpClient.listFiles("remote-test-dir/"))
.thenReturn(ftpFiles.toArray(new FTPFile[ftpFiles.size()]));
.thenReturn(ftpFiles.toArray(new FTPFile[0]));
when(ftpClient.getRemoteAddress())
.thenReturn(InetAddress.getByName("127.0.0.1"));
return ftpClient;
}
catch (Exception e) {