From 72dd39bccf5ac235deb2a4926a41895271d899b6 Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Wed, 6 Nov 2013 19:44:52 -0500 Subject: [PATCH 01/29] INT-2898 Add Persistent FileListFilters - Abstract implementation - Implementation for the file system - Implementation for remote files ((S)FTP) JIRA: https://jira.springsource.org/browse/INT-2898 INT-2889 Polishing - Add (S)FTP test cases - Docs --- .../PropertiesPersistingMetadataStore.java | 16 ++- ...actPersistentAcceptOnceFileListFilter.java | 92 +++++++++++++ ...temPersistentAcceptOnceFileListFilter.java | 44 ++++++ ...rsistentFilterIntegrationTests-context.xml | 26 ++++ ...ourcePersistentFilterIntegrationTests.java | 126 ++++++++++++++++++ ...rsistentAcceptOnceFileListFilterTests.java | 47 +++++++ ...FtpPersistentAcceptOnceFileListFilter.java | 49 +++++++ ...oundRemoteFileSystemSynchronizerTests.java | 57 ++++++-- ...ftpPersistentAcceptOnceFileListFilter.java | 51 +++++++ ...oundRemoteFileSystemSynchronizerTests.java | 62 +++++++-- src/reference/docbook/file.xml | 8 ++ src/reference/docbook/ftp.xml | 35 ++++- src/reference/docbook/sftp.xml | 35 ++++- src/reference/docbook/whats-new.xml | 8 ++ 14 files changed, 618 insertions(+), 38 deletions(-) create mode 100644 spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java create mode 100644 spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemPersistentAcceptOnceFileListFilter.java create mode 100644 spring-integration-file/src/test/java/org/springframework/integration/file/FileReadingMessageSourcePersistentFilterIntegrationTests-context.xml create mode 100644 spring-integration-file/src/test/java/org/springframework/integration/file/FileReadingMessageSourcePersistentFilterIntegrationTests.java create mode 100644 spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterTests.java create mode 100644 spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpPersistentAcceptOnceFileListFilter.java create mode 100644 spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpPersistentAcceptOnceFileListFilter.java diff --git a/spring-integration-core/src/main/java/org/springframework/integration/metadata/PropertiesPersistingMetadataStore.java b/spring-integration-core/src/main/java/org/springframework/integration/metadata/PropertiesPersistingMetadataStore.java index 26c35231ec..b6599cef0d 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/metadata/PropertiesPersistingMetadataStore.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/metadata/PropertiesPersistingMetadataStore.java @@ -28,6 +28,7 @@ import java.util.Properties; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; + import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.InitializingBean; import org.springframework.util.Assert; @@ -62,6 +63,7 @@ public class PropertiesPersistingMetadataStore implements MetadataStore, Initial this.baseDirectory = baseDirectory; } + @Override public void afterPropertiesSet() throws Exception { File baseDir = new File(baseDirectory); baseDir.mkdirs(); @@ -78,20 +80,22 @@ public class PropertiesPersistingMetadataStore implements MetadataStore, Initial this.loadMetadata(); } + @Override public void put(String key, String value) { this.metadata.setProperty(key, value); } + @Override public String get(String key) { return this.metadata.getProperty(key); } @Override - @SuppressWarnings("uchecked") public String remove(String key) { return (String) this.metadata.remove(key); } + @Override public void destroy() throws Exception { this.saveMetadata(); } @@ -100,12 +104,12 @@ public class PropertiesPersistingMetadataStore implements MetadataStore, Initial OutputStream outputStream = null; try { outputStream = new BufferedOutputStream(new FileOutputStream(this.file)); - this.persister.store(this.metadata, outputStream, "Last feed entry"); + this.persister.store(this.metadata, outputStream, "Last entry"); } catch (IOException e) { // not fatal for the functionality of the component - logger.warn("Failed to persist feed entry. This may result in a duplicate " - + "feed entry after this component is restarted.", e); + logger.warn("Failed to persist entry. This may result in a duplicate " + + "entry after this component is restarted.", e); } finally { try { @@ -128,8 +132,8 @@ public class PropertiesPersistingMetadataStore implements MetadataStore, Initial } catch (Exception e) { // not fatal for the functionality of the component - logger.warn("Failed to load feed entry from the persistent store. This may result in a duplicate " + - "feed entry after this component is restarted", e); + logger.warn("Failed to load entry from the persistent store. This may result in a duplicate " + + "entry after this component is restarted", e); } finally { try { diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java new file mode 100644 index 0000000000..0f752473f5 --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java @@ -0,0 +1,92 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.file.filters; + +import org.springframework.integration.metadata.MetadataStore; +import org.springframework.util.Assert; + +/** + * Stores "seen" files in a MetadataStore to survive application restarts. + * The default key is 'prefix' plus the absolute file name; value is the timestamp of the file. + * Files are deemed as already 'seen' if they exist in the store and have the + * same modified time as the current file. + * + * @author Gary Russell + * @since 3.0 + * + */ +public abstract class AbstractPersistentAcceptOnceFileListFilter extends AbstractFileListFilter { + + protected final MetadataStore store; + + protected final String prefix; + + private final Object monitor = new Object(); + + public AbstractPersistentAcceptOnceFileListFilter(MetadataStore store, String prefix) { + Assert.notNull(store, "'store' cannot be null"); + Assert.notNull(prefix, "'prefix' cannot be null"); + this.store = store; + this.prefix = prefix; + } + + @Override + protected boolean accept(F file) { + String key = buildKey(file); + synchronized(monitor) { + String value = store.get(key); + if (value != null && isEqual(file, value)) { + return false; + } + store.put(key, value(file)); + } + return true; + } + + /** + * The default value stored for the key is the last modified date. + * @param file The file. + * @return The value to store for the file. + */ + private String value(F file) { + return Long.toString(this.modified(file)); + } + + /** + * Override this method if you wish to use something other than the + * modified timestamp to determine equality. + * @param file The file. + * @param value The current value for the key in the store + * @return + */ + protected boolean isEqual(F file, String value) { + return Long.valueOf(value).longValue() == this.modified(file); + } + + /** + * The default key is the {@link #prefix} plus the full filename. + * @param file The file. + * @return The key. + */ + protected String buildKey(F file) { + return this.prefix + this.fileName(file); + } + + protected abstract long modified(F file); + + protected abstract String fileName(F file); + +} diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemPersistentAcceptOnceFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemPersistentAcceptOnceFileListFilter.java new file mode 100644 index 0000000000..6fb2a2c19f --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemPersistentAcceptOnceFileListFilter.java @@ -0,0 +1,44 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.file.filters; + + +import java.io.File; + +import org.springframework.integration.metadata.MetadataStore; + +/** + * @author Gary Russell + * @since 3.0 + * + */ +public class FileSystemPersistentAcceptOnceFileListFilter extends AbstractPersistentAcceptOnceFileListFilter { + + public FileSystemPersistentAcceptOnceFileListFilter(MetadataStore store, String prefix) { + super(store, prefix); + } + + @Override + protected long modified(File file) { + return file.lastModified(); + } + + @Override + protected String fileName(File file) { + return file.getAbsolutePath(); + } + +} diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/FileReadingMessageSourcePersistentFilterIntegrationTests-context.xml b/spring-integration-file/src/test/java/org/springframework/integration/file/FileReadingMessageSourcePersistentFilterIntegrationTests-context.xml new file mode 100644 index 0000000000..1292e3905a --- /dev/null +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/FileReadingMessageSourcePersistentFilterIntegrationTests-context.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/FileReadingMessageSourcePersistentFilterIntegrationTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/FileReadingMessageSourcePersistentFilterIntegrationTests.java new file mode 100644 index 0000000000..293738d1c5 --- /dev/null +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/FileReadingMessageSourcePersistentFilterIntegrationTests.java @@ -0,0 +1,126 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.file; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; + +import java.io.File; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.context.support.AbstractApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; +import org.springframework.integration.Message; + +/** + * @author Iwein Fuld + * @author Gary Russell + */ +public class FileReadingMessageSourcePersistentFilterIntegrationTests { + + AbstractApplicationContext context; + + FileReadingMessageSource pollableFileSource; + + private static File inputDir; + + @AfterClass + public static void cleanUp() throws Throwable { + if(inputDir.exists()) { + inputDir.delete(); + } + } + + @BeforeClass + public static void setupInputDir() { + inputDir = new File(System.getProperty("java.io.tmpdir") + "/" + + FileReadingMessageSourcePersistentFilterIntegrationTests.class.getSimpleName()); + inputDir.mkdir(); + } + + @Before + public void generateTestFiles() throws Exception { + File.createTempFile("test", null, inputDir).setLastModified(System.currentTimeMillis() - 1000); + File.createTempFile("test", null, inputDir).setLastModified(System.currentTimeMillis() - 1000); + File.createTempFile("test", null, inputDir).setLastModified(System.currentTimeMillis() - 1000); + this.loadContextAndGetMessageSource(); + } + + private void loadContextAndGetMessageSource() { + this.context = new ClassPathXmlApplicationContext(this.getClass().getSimpleName() + "-context.xml", + this.getClass()); + this.pollableFileSource = context.getBean(FileReadingMessageSource.class); + } + + @After + public void cleanoutInputDir() throws Exception { + File[] listFiles = inputDir.listFiles(); + for (int i = 0; i < listFiles.length; i++) { + listFiles[i].delete(); + } + } + + @AfterClass + public static void removeInputDir() throws Exception { + inputDir.delete(); + File persistDir = new File(System.getProperty("java.io.tmpdir") + "/" + + FileReadingMessageSourcePersistentFilterIntegrationTests.class.getSimpleName() + + ".meta"); + File persist = new File(persistDir, "metadata-store.properties"); + persist.delete(); + persistDir.delete(); + } + + + @Test + public void configured() throws Exception { + DirectFieldAccessor accessor = new DirectFieldAccessor(pollableFileSource); + assertEquals(inputDir, accessor.getPropertyValue("directory")); + } + + @Test + public void getFiles() throws Exception { + Message received1 = pollableFileSource.receive(); + System.out.println("receive files round 1"); + assertNotNull("This should return the first message", received1); + pollableFileSource.onSend(received1); + Message received2 = pollableFileSource.receive(); + assertNotNull(received2); + pollableFileSource.onSend(received2); + Message received3 = pollableFileSource.receive(); + assertNotNull(received3); + pollableFileSource.onSend(received3); + assertNotSame(received1 + " == " + received2, received1.getPayload(), received2.getPayload()); + assertNotSame(received1 + " == " + received3, received1.getPayload(), received3.getPayload()); + assertNotSame(received2 + " == " + received3, received2.getPayload(), received3.getPayload()); + this.context.destroy(); + + this.loadContextAndGetMessageSource(); + Message received4 = pollableFileSource.receive(); + assertNull(received4); + this.context.destroy(); + } + +} diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterTests.java new file mode 100644 index 0000000000..0fe1e51dbd --- /dev/null +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterTests.java @@ -0,0 +1,47 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.file.filters; + +import static org.junit.Assert.assertTrue; + +import java.io.File; + +import org.junit.Test; + +import org.springframework.integration.metadata.MetadataStore; +import org.springframework.integration.metadata.SimpleMetadataStore; + +/** + * @author Gary Russell + * @since 3.0 + * + */ +public class PersistentAcceptOnceFileListFilterTests { + + @Test + public void testFileSystem() throws Exception { + MetadataStore store = new SimpleMetadataStore(); + FileSystemPersistentAcceptOnceFileListFilter filter = new FileSystemPersistentAcceptOnceFileListFilter(store, "foo:"); + File file = File.createTempFile("foo", ".txt"); + assertTrue(filter.filterFiles(new File[] {file}).size() == 1); + assertTrue(filter.filterFiles(new File[] {file}).size() == 0); + file.setLastModified(27L); + assertTrue(filter.filterFiles(new File[] {file}).size() == 1); + assertTrue(filter.filterFiles(new File[] {file}).size() == 0); + file.delete(); + } + +} diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpPersistentAcceptOnceFileListFilter.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpPersistentAcceptOnceFileListFilter.java new file mode 100644 index 0000000000..91ae1bd3f3 --- /dev/null +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpPersistentAcceptOnceFileListFilter.java @@ -0,0 +1,49 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.ftp.filters; + + +import org.apache.commons.net.ftp.FTPFile; + +import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter; +import org.springframework.integration.metadata.MetadataStore; + +/** + * Since the super class deems files as 'not seen' if the timestamp is different, remote file + * users should use the adapter's preserve-timestamp option. Otherwise if a file is re-fetched + * it will have a new timestamp. + * + * @author Gary Russell + * @since 3.0 + * + */ +public class FtpPersistentAcceptOnceFileListFilter extends AbstractPersistentAcceptOnceFileListFilter { + + public FtpPersistentAcceptOnceFileListFilter(MetadataStore store, String prefix) { + super(store, prefix); + } + + @Override + protected long modified(FTPFile file) { + return file.getTimestamp().getTimeInMillis(); + } + + @Override + protected String fileName(FTPFile file) { + return file.getName(); + } + +} diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpInboundRemoteFileSystemSynchronizerTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpInboundRemoteFileSystemSynchronizerTests.java index 7106219858..51e24f9cac 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpInboundRemoteFileSystemSynchronizerTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpInboundRemoteFileSystemSynchronizerTests.java @@ -33,11 +33,14 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; +import java.util.List; +import java.util.Queue; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPFile; import org.hamcrest.Matchers; import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -47,8 +50,13 @@ import org.springframework.expression.spel.SpelParserConfiguration; import org.springframework.expression.spel.standard.SpelExpressionParser; import org.springframework.integration.Message; import org.springframework.integration.expression.ExpressionUtils; +import org.springframework.integration.file.filters.CompositeFileListFilter; +import org.springframework.integration.file.filters.FileListFilter; +import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter; import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter; import org.springframework.integration.ftp.session.AbstractFtpSessionFactory; +import org.springframework.integration.metadata.PropertiesPersistingMetadataStore; +import org.springframework.integration.test.util.TestUtils; /** * @author Oleg Zhurakousky @@ -61,6 +69,7 @@ public class FtpInboundRemoteFileSystemSynchronizerTests { private static FTPClient ftpClient = mock(FTPClient.class); + @Before @After public void cleanup(){ File file = new File("test"); @@ -86,7 +95,16 @@ public class FtpInboundRemoteFileSystemSynchronizerTests { synchronizer.setDeleteRemoteFiles(true); synchronizer.setPreserveTimestamp(true); synchronizer.setRemoteDirectory("remote-test-dir"); - synchronizer.setFilter(new FtpRegexPatternFileListFilter(".*\\.test$")); + FtpRegexPatternFileListFilter patternFilter = new FtpRegexPatternFileListFilter(".*\\.test$"); + PropertiesPersistingMetadataStore store = new PropertiesPersistingMetadataStore(); + store.setBaseDirectory("test"); + FtpPersistentAcceptOnceFileListFilter persistFilter = + new FtpPersistentAcceptOnceFileListFilter(store, "foo"); + List> filters = new ArrayList>(); + filters.add(persistFilter); + filters.add(patternFilter); + CompositeFileListFilter filter = new CompositeFileListFilter(filters); + synchronizer.setFilter(filter); synchronizer.setIntegrationEvaluationContext(ExpressionUtils.createStandardEvaluationContext()); ExpressionParser expressionParser = new SpelExpressionParser(new SpelParserConfiguration(true, true)); @@ -120,28 +138,49 @@ public class FtpInboundRemoteFileSystemSynchronizerTests { assertTrue(new File("test/A.TEST.a").exists()); assertTrue(new File("test/B.TEST.a").exists()); + + TestUtils.getPropertyValue(ms, "localFileListFilter.seen", Queue.class).clear(); + + new File("test/A.TEST.a").delete(); + new File("test/B.TEST.a").delete(); + // the remote filter should prevent a re-fetch + nothing = ms.receive(); + assertNull(nothing); + } public static class TestFtpSessionFactory extends AbstractFtpSessionFactory { + private final Collection ftpFiles = new ArrayList(); + + private void init() { + String[] files = new File("remote-test-dir").list(); + for (String fileName : files) { + FTPFile file = new FTPFile(); + file.setName(fileName); + file.setType(FTPFile.FILE_TYPE); + Calendar calendar = Calendar.getInstance(); + calendar.add(Calendar.DATE, 1); + file.setTimestamp(calendar); + ftpFiles.add(file); + } + + } + @Override protected FTPClient createClientInstance() { + if (this.ftpFiles.size() == 0) { + this.init(); + } + try { when(ftpClient.getReplyCode()).thenReturn(250); when(ftpClient.login("kermit", "frog")).thenReturn(true); when(ftpClient.changeWorkingDirectory(Mockito.anyString())).thenReturn(true); String[] files = new File("remote-test-dir").list(); - Collection ftpFiles = new ArrayList(); for (String fileName : files) { - FTPFile file = new FTPFile(); - file.setName(fileName); - file.setType(FTPFile.FILE_TYPE); - Calendar calendar = Calendar.getInstance(); - calendar.add(Calendar.DATE, 1); - file.setTimestamp(calendar); - ftpFiles.add(file); when(ftpClient.retrieveFile(Mockito.eq("remote-test-dir/" + fileName) , Mockito.any(OutputStream.class))).thenReturn(true); } when(ftpClient.listFiles("remote-test-dir")).thenReturn(ftpFiles.toArray(new FTPFile[]{})); diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpPersistentAcceptOnceFileListFilter.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpPersistentAcceptOnceFileListFilter.java new file mode 100644 index 0000000000..0140c2d8e2 --- /dev/null +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpPersistentAcceptOnceFileListFilter.java @@ -0,0 +1,51 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.sftp.filters; + + + +import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter; +import org.springframework.integration.metadata.MetadataStore; + +import com.jcraft.jsch.ChannelSftp.LsEntry; + +/** + * Since the super class deems files as 'not seen' if the timestamp is different, remote file + * users should use the adapter's preserve-timestamp option. Otherwise if a file is re-fetched + * it will have a new timestamp. + * + * @author Gary Russell + * @since 3.0 + * + */ +public class SftpPersistentAcceptOnceFileListFilter extends AbstractPersistentAcceptOnceFileListFilter { + + public SftpPersistentAcceptOnceFileListFilter(MetadataStore store, String prefix) { + super(store, prefix); + } + + @Override + protected long modified(LsEntry file) { + return ((long) file.getAttrs().getMTime()) * 1000; + } + + @Override + protected String fileName(LsEntry file) { + return file.getLongname(); + } + + +} diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpInboundRemoteFileSystemSynchronizerTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpInboundRemoteFileSystemSynchronizerTests.java index c8fa674f48..516cadee84 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpInboundRemoteFileSystemSynchronizerTests.java +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpInboundRemoteFileSystemSynchronizerTests.java @@ -30,19 +30,28 @@ import static org.mockito.Mockito.when; import java.io.File; import java.io.FileInputStream; +import java.util.ArrayList; import java.util.Calendar; +import java.util.List; +import java.util.Queue; import java.util.Vector; import org.hamcrest.Matchers; import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.springframework.integration.Message; import org.springframework.integration.expression.ExpressionUtils; +import org.springframework.integration.file.filters.CompositeFileListFilter; +import org.springframework.integration.file.filters.FileListFilter; import org.springframework.integration.file.remote.session.Session; +import org.springframework.integration.metadata.PropertiesPersistingMetadataStore; +import org.springframework.integration.sftp.filters.SftpPersistentAcceptOnceFileListFilter; import org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter; import org.springframework.integration.sftp.session.DefaultSftpSessionFactory; import org.springframework.integration.sftp.session.SftpTestSessionFactory; +import org.springframework.integration.test.util.TestUtils; import com.jcraft.jsch.ChannelSftp; import com.jcraft.jsch.ChannelSftp.LsEntry; @@ -59,6 +68,7 @@ public class SftpInboundRemoteFileSystemSynchronizerTests { private static com.jcraft.jsch.Session jschSession = mock(com.jcraft.jsch.Session.class); + @Before @After public void cleanup(){ File file = new File("test"); @@ -87,7 +97,16 @@ public class SftpInboundRemoteFileSystemSynchronizerTests { synchronizer.setDeleteRemoteFiles(true); synchronizer.setPreserveTimestamp(true); synchronizer.setRemoteDirectory("remote-test-dir"); - synchronizer.setFilter(new SftpRegexPatternFileListFilter(".*\\.test$")); + SftpRegexPatternFileListFilter patternFilter = new SftpRegexPatternFileListFilter(".*\\.test$"); + PropertiesPersistingMetadataStore store = new PropertiesPersistingMetadataStore(); + store.setBaseDirectory("test"); + SftpPersistentAcceptOnceFileListFilter persistFilter = + new SftpPersistentAcceptOnceFileListFilter(store, "foo"); + List> filters = new ArrayList>(); + filters.add(persistFilter); + filters.add(patternFilter); + CompositeFileListFilter filter = new CompositeFileListFilter(filters); + synchronizer.setFilter(filter); synchronizer.setIntegrationEvaluationContext(ExpressionUtils.createStandardEvaluationContext()); SftpInboundFileSynchronizingMessageSource ms = @@ -115,28 +134,48 @@ public class SftpInboundRemoteFileSystemSynchronizerTests { assertTrue(new File("test/a.test").exists()); assertTrue(new File("test/b.test").exists()); + + TestUtils.getPropertyValue(ms, "localFileListFilter.seen", Queue.class).clear(); + + new File("test/a.test").delete(); + new File("test/b.test").delete(); + // the remote filter should prevent a re-fetch + nothing = ms.receive(); + assertNull(nothing); + } public static class TestSftpSessionFactory extends DefaultSftpSessionFactory { + private final Vector sftpEntries = new Vector(); + + private void init() { + String[] files = new File("remote-test-dir").list(); + for (String fileName : files) { + LsEntry lsEntry = mock(LsEntry.class); + SftpATTRS attributes = mock(SftpATTRS.class); + when(lsEntry.getAttrs()).thenReturn(attributes); + + Calendar calendar = Calendar.getInstance(); + calendar.add(Calendar.DATE, 1); + when(lsEntry.getAttrs().getMTime()).thenReturn(new Long(calendar.getTimeInMillis() / 1000).intValue()); + when(lsEntry.getFilename()).thenReturn(fileName); + when(lsEntry.getLongname()).thenReturn(fileName); + sftpEntries.add(lsEntry); + } + } @Override public Session getSession() { + if (this.sftpEntries.size() == 0) { + this.init(); + } + try { ChannelSftp channel = mock(ChannelSftp.class); String[] files = new File("remote-test-dir").list(); - Vector sftpEntries = new Vector(); for (String fileName : files) { - LsEntry lsEntry = mock(LsEntry.class); - SftpATTRS attributes = mock(SftpATTRS.class); - when(lsEntry.getAttrs()).thenReturn(attributes); - - Calendar calendar = Calendar.getInstance(); - calendar.add(Calendar.DATE, 1); - when(lsEntry.getAttrs().getMTime()).thenReturn(new Long(calendar.getTimeInMillis() / 1000).intValue()); - when(lsEntry.getFilename()).thenReturn(fileName); - sftpEntries.add(lsEntry); when(channel.get("remote-test-dir/"+fileName)).thenReturn(new FileInputStream("remote-test-dir/" + fileName)); } when(channel.ls("remote-test-dir")).thenReturn(sftpEntries); @@ -148,4 +187,5 @@ public class SftpInboundRemoteFileSystemSynchronizerTests { } } } + } diff --git a/src/reference/docbook/file.xml b/src/reference/docbook/file.xml index b1f2c46897..9cc835c8e2 100644 --- a/src/reference/docbook/file.xml +++ b/src/reference/docbook/file.xml @@ -35,6 +35,14 @@ FileListFilter. By default, an AcceptOnceFileListFilter is used. This filter ensures files are picked up only once from the directory. + + The AcceptOnceFileListFilter stores its state in memory. If you wish the + state to survive a system restart, consider using the + FileSystemPersistentAcceptOnceFileListFilter instead. This filter stores + the accepted file names in a MetadataStore. The framework supplies + several store implementations (such as Redis), or you can provide your own. This filter matches on + the filename and modified time. + filename-regex=".*\.test$"). And of course if you need complete control you can use filter attribute and provide a reference to any custom implementation of the org.springframework.integration.file.filters.FileListFilter, a strategy interface for filtering a - list of files. This filter determines which remote files are retrieved. + list of files. This filter determines which remote files are retrieved. You can also combine a pattern based filter + with other filters, such as an AcceptOnceFileListFilter to avoid synchronizing files that + have previously been fetched, by using a CompositeFileListFilter. + + The AcceptOnceFileListFilter stores its state in memory. If you wish the + state to survive a system restart, consider using the + FtpPersistentAcceptOnceFileListFilter instead. This filter stores + the accepted file names in a MetadataStore. The framework supplies + several store implementations (such as Redis), or you can provide your own. This filter matches on + the filename and the remote modified time. + - Beginning with 3.0, you can also specify a filter used to filter the files locally, once they have - been retrieved. The default filter is an AcceptOnceFileListFilter which prevents processing - files with the same name multiple times in the same JVM execution; this can now be overridden - (for example with an AcceptAllFileListFilter), using the local-filter attribute. - Previously, the default AcceptOnceFileListFilter could not be overridden. + + Beginning with version 3.0, you can also specify a filter used to filter the files locally, once they have + been retrieved. The default filter is an AcceptOnceFileListFilter which prevents processing + files with the same name multiple times in the same JVM execution; this can now be overridden + (for example with an AcceptAllFileListFilter), using the local-filter attribute. + Previously, the default AcceptOnceFileListFilter could not be overridden. + + + The AcceptOnceFileListFilter stores its state in memory. If you wish the + state to survive a system restart, consider using the + FileSystemPersistentAcceptOnceFileListFilter as a local filter instead. This filter stores + the accepted file names in a MetadataStore. The framework supplies + several store implementations (such as Redis), or you can provide your own. + + This filter compares the filename and modified timestamp. If you wish to use this technique to avoid a + re-synchronized file from being processed, you should use the preserve-timestamp attribute discussed above. + + The 'remote-file-separator' attribute allows you to configure a diff --git a/src/reference/docbook/sftp.xml b/src/reference/docbook/sftp.xml index 5111c75d5f..0d8c56a677 100644 --- a/src/reference/docbook/sftp.xml +++ b/src/reference/docbook/sftp.xml @@ -297,14 +297,37 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp (e.g. filename-regex=".*\.test$"). And of course if you need complete control you can use the filter attribute to provide a reference to a custom implementation of the org.springframework.integration.file.filters.FileListFilter - a strategy interface for filtering a - list of files. This filter determines which remote files are retrieved. + list of files. This filter determines which remote files are retrieved. You can also combine a pattern based filter + with other filters, such as an AcceptOnceFileListFilter to avoid synchronizing files that + have previously been fetched, by using a CompositeFileListFilter. + + The AcceptOnceFileListFilter stores its state in memory. If you wish the + state to survive a system restart, consider using the + SftpPersistentAcceptOnceFileListFilter instead. This filter stores + the accepted file names in a MetadataStore. The framework supplies + several store implementations (such as Redis), or you can provide your own. This filter matches on + the filename and the remote modified time. + - Beginning with 3.0, you can also specify a filter used to filter the files locally, once they have - been retrieved. The default filter is an AcceptOnceFileListFilter which prevents processing - files with the same name multiple times in the same JVM execution; this can now be overridden - (for example with an AcceptAllFileListFilter), using the local-filter attribute. - Previously, the default AcceptOnceFileListFilter could not be overridden. + + Beginning with version 3.0, you can also specify a filter used to filter the files locally, once they have + been retrieved. The default filter is an AcceptOnceFileListFilter which prevents processing + files with the same name multiple times in the same JVM execution; this can now be overridden + (for example with an AcceptAllFileListFilter), using the local-filter attribute. + Previously, the default AcceptOnceFileListFilter could not be overridden. + + + The AcceptOnceFileListFilter stores its state in memory. If you wish the + state to survive a system restart, consider using the + FileSystemPersistentAcceptOnceFileListFilter as a local filter instead. This filter stores + the accepted file names in a MetadataStore. The framework supplies + several store implementations (such as Redis), or you can provide your own. + + This filter compares the filename and modified timestamp. If you wish to use this technique to avoid a + re-synchronized file from being processed, you should use the preserve-timestamp attribute discussed above. + + Please refer to the schema for more detail on these attributes. diff --git a/src/reference/docbook/whats-new.xml b/src/reference/docbook/whats-new.xml index c1271ca497..c10f243959 100644 --- a/src/reference/docbook/whats-new.xml +++ b/src/reference/docbook/whats-new.xml @@ -604,5 +604,13 @@ For more information, see . +
+ Persistend File List Filters (file, (S)FTP) + + New FileListFilters that use a persistent MetadataStore are + now available. These can be used to prevent duplicate files after a system restart. See + , , and for more information. + +
From d662e7ee422c8a8cfe1de5b7ecdda7124e6d4446 Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Mon, 11 Nov 2013 17:53:50 -0500 Subject: [PATCH 02/29] INT-3203 Introduce RemoteFileTemplate Begin by refactoring `FileTransferringMessageHandler` to use a `send` operation on RemoteFileTemplate. Effectively transfer all properties from the adapter to the template. This will allow reuse of the send logic from the adapter in the outbound gateway for `put` and `mput`. JIRA: https://jira.springsource.org/browse/INT-3203 --- .../file/remote/RemoteFileOperations.java | 29 ++ .../file/remote/RemoteFileTemplate.java | 311 ++++++++++++++++++ .../FileTransferringMessageHandler.java | 232 +------------ .../FtpOutboundChannelAdapterParserTests.java | 24 +- ...FtpsOutboundChannelAdapterParserTests.java | 6 +- .../ftp/outbound/FtpOutboundTests.java | 12 +- ...boundChannelAdapterParserCachingTests.java | 4 +- .../OutboundChannelAdapterParserTests.java | 24 +- 8 files changed, 395 insertions(+), 247 deletions(-) create mode 100644 spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java create mode 100644 spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java new file mode 100644 index 0000000000..1b61d9cb23 --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java @@ -0,0 +1,29 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.file.remote; + +import org.springframework.integration.Message; + +/** + * @author Gary Russell + * @since 3.0 + * + */ +public interface RemoteFileOperations { + + public abstract void send(Message message) throws Exception; + +} diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java new file mode 100644 index 0000000000..f96fe3697d --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java @@ -0,0 +1,311 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.file.remote; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.BeanFactoryAware; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.expression.Expression; +import org.springframework.integration.Message; +import org.springframework.integration.MessageDeliveryException; +import org.springframework.integration.MessagingException; +import org.springframework.integration.file.DefaultFileNameGenerator; +import org.springframework.integration.file.FileNameGenerator; +import org.springframework.integration.file.remote.session.Session; +import org.springframework.integration.file.remote.session.SessionFactory; +import org.springframework.integration.handler.ExpressionEvaluatingMessageProcessor; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; + +/** + * @author Iwein Fuld + * @author Mark Fisher + * @author Josh Long + * @author Oleg Zhurakousky + * @author David Turanski + * @author Gary Russell + * @since 3.0 + * + */ +public class RemoteFileTemplate implements RemoteFileOperations, InitializingBean, BeanFactoryAware { + + private final Log logger = LogFactory.getLog(this.getClass()); + + private final SessionFactory sessionFactory; + + private volatile String temporaryFileSuffix =".writing"; + + private volatile boolean autoCreateDirectory = false; + + private volatile boolean useTemporaryFileName = true; + + private volatile ExpressionEvaluatingMessageProcessor directoryExpressionProcessor; + + private volatile ExpressionEvaluatingMessageProcessor temporaryDirectoryExpressionProcessor; + + private volatile FileNameGenerator fileNameGenerator = new DefaultFileNameGenerator(); + + private volatile boolean fileNameGeneratorSet; + + private volatile String charset = "UTF-8"; + + private volatile String remoteFileSeparator = "/"; + + private volatile boolean hasExplicitlySetSuffix; + + private volatile BeanFactory beanFactory; + + public RemoteFileTemplate(SessionFactory sessionFactory) { + Assert.notNull(sessionFactory, "sessionFactory must not be null"); + this.sessionFactory = sessionFactory; + } + + public void setAutoCreateDirectory(boolean autoCreateDirectory) { + this.autoCreateDirectory = autoCreateDirectory; + } + + public void setRemoteFileSeparator(String remoteFileSeparator) { + Assert.notNull(remoteFileSeparator, "'remoteFileSeparator' must not be null"); + this.remoteFileSeparator = remoteFileSeparator; + } + + public void setRemoteDirectoryExpression(Expression remoteDirectoryExpression) { + Assert.notNull(remoteDirectoryExpression, "remoteDirectoryExpression must not be null"); + this.directoryExpressionProcessor = new ExpressionEvaluatingMessageProcessor(remoteDirectoryExpression, String.class); + } + + public void setTemporaryRemoteDirectoryExpression(Expression temporaryRemoteDirectoryExpression) { + Assert.notNull(temporaryRemoteDirectoryExpression, "temporaryRemoteDirectoryExpression must not be null"); + this.temporaryDirectoryExpressionProcessor = new ExpressionEvaluatingMessageProcessor(temporaryRemoteDirectoryExpression, String.class); + } + + public String getTemporaryFileSuffix() { + return this.temporaryFileSuffix; + } + + public boolean isUseTemporaryFileName() { + return useTemporaryFileName; + } + + public void setUseTemporaryFileName(boolean useTemporaryFileName) { + this.useTemporaryFileName = useTemporaryFileName; + } + + public void setFileNameGenerator(FileNameGenerator fileNameGenerator) { + this.fileNameGenerator = (fileNameGenerator != null) ? fileNameGenerator : new DefaultFileNameGenerator(); + this.fileNameGeneratorSet = fileNameGenerator != null; + } + + public void setCharset(String charset) { + this.charset = charset; + } + + public void setTemporaryFileSuffix(String temporaryFileSuffix) { + Assert.notNull(temporaryFileSuffix, "'temporaryFileSuffix' must not be null"); + this.hasExplicitlySetSuffix = true; + this.temporaryFileSuffix = temporaryFileSuffix; + } + + + @Override + public void setBeanFactory(BeanFactory beanFactory) throws BeansException { + this.beanFactory = beanFactory; + } + + @Override + public void afterPropertiesSet() throws Exception { + Assert.notNull(this.directoryExpressionProcessor, "remoteDirectoryExpression is required"); + BeanFactory beanFactory = this.beanFactory; + if (beanFactory != null) { + this.directoryExpressionProcessor.setBeanFactory(beanFactory); + if (this.temporaryDirectoryExpressionProcessor != null) { + this.temporaryDirectoryExpressionProcessor.setBeanFactory(beanFactory); + } + if (!this.fileNameGeneratorSet && this.fileNameGenerator instanceof BeanFactoryAware) { + ((BeanFactoryAware) this.fileNameGenerator).setBeanFactory(beanFactory); + } + } + if (this.autoCreateDirectory){ + Assert.hasText(this.remoteFileSeparator, "'remoteFileSeparator' must not be empty when 'autoCreateDirectory' is set to 'true'"); + } + if (hasExplicitlySetSuffix && !useTemporaryFileName){ + this.logger.warn("Since 'use-temporary-file-name' is set to 'false' the value of 'temporary-file-suffix' has no effect"); + } + } + + @Override + public void send(Message message) throws Exception { + StreamHolder inputStreamHolder = this.payloadToInputStream(message); + if (inputStreamHolder != null) { + Session session = this.sessionFactory.getSession(); + String fileName = inputStreamHolder.getName(); + try { + String remoteDirectory = this.directoryExpressionProcessor.processMessage(message); + String temporaryRemoteDirectory = remoteDirectory; + if (this.temporaryDirectoryExpressionProcessor != null){ + temporaryRemoteDirectory = this.temporaryDirectoryExpressionProcessor.processMessage(message); + } + fileName = this.fileNameGenerator.generateFileName(message); + this.sendFileToRemoteDirectory(inputStreamHolder.getStream(), temporaryRemoteDirectory, remoteDirectory, fileName, session); + } + catch (FileNotFoundException e) { + throw new MessageDeliveryException(message, + "File [" + inputStreamHolder.getName() + "] not found in local working directory; it was moved or deleted unexpectedly.", e); + } + catch (IOException e) { + throw new MessageDeliveryException(message, + "Failed to transfer file [" + inputStreamHolder.getName() + " -> " + fileName + "] from local directory to remote directory.", e); + } + catch (Exception e) { + throw new MessageDeliveryException(message, + "Error handling message for file [" + inputStreamHolder.getName() + " -> " + fileName + "]", e); + } + finally { + if (session != null) { + session.close(); + } + } + } + else { + // A null holder means a File payload that does not exist. + if (logger.isWarnEnabled()) { + logger.warn("File " + message.getPayload() + " does not exist"); + } + } + } + + private StreamHolder payloadToInputStream(Message message) throws MessageDeliveryException { + try { + Object payload = message.getPayload(); + InputStream dataInputStream = null; + String name = null; + if (payload instanceof File) { + File inputFile = (File) payload; + if (inputFile.exists()) { + dataInputStream = new BufferedInputStream(new FileInputStream(inputFile)); + name = inputFile.getAbsolutePath(); + } + } + else if (payload instanceof byte[] || payload instanceof String) { + byte[] bytes = null; + if (payload instanceof String) { + bytes = ((String) payload).getBytes(this.charset); + name = "String payload"; + } + else { + bytes = (byte[]) payload; + name = "byte[] payload"; + } + dataInputStream = new ByteArrayInputStream(bytes); + } + else { + throw new IllegalArgumentException("Unsupported payload type. The only supported payloads are " + + "java.io.File, java.lang.String, and byte[]"); + } + if (dataInputStream == null) { + return null; + } + else { + return new StreamHolder(dataInputStream, name); + } + } + catch (Exception e) { + throw new MessageDeliveryException(message, "Failed to create sendable file.", e); + } + } + + private void sendFileToRemoteDirectory(InputStream inputStream, String temporaryRemoteDirectory, + String remoteDirectory, String fileName, Session session) throws FileNotFoundException, IOException { + + remoteDirectory = this.normalizeDirectoryPath(remoteDirectory); + temporaryRemoteDirectory = this.normalizeDirectoryPath(temporaryRemoteDirectory); + + String remoteFilePath = remoteDirectory + fileName; + String tempRemoteFilePath = temporaryRemoteDirectory + fileName; + // write remote file first with temporary file extension if enabled + + String tempFilePath = tempRemoteFilePath + (useTemporaryFileName ? this.temporaryFileSuffix : ""); + + if (this.autoCreateDirectory) { + try { + RemoteFileUtils.makeDirectories(remoteDirectory, session, this.remoteFileSeparator, this.logger); + } + catch (IllegalStateException e) { + // Revert to old FTP behavior if recursive mkdir fails, for backwards compatibility + session.mkdir(remoteDirectory); + } + } + + try { + session.write(inputStream, tempFilePath); + // then rename it to its final name if necessary + if (useTemporaryFileName){ + session.rename(tempFilePath, remoteFilePath); + } + } + catch (Exception e) { + throw new MessagingException("Failed to write to '" + tempFilePath + "' while uploading the file", e); + } + finally { + inputStream.close(); + } + } + + private String normalizeDirectoryPath(String directoryPath){ + if (!StringUtils.hasText(directoryPath)) { + directoryPath = ""; + } + else if (!directoryPath.endsWith(this.remoteFileSeparator)) { + directoryPath += this.remoteFileSeparator; + } + return directoryPath; + } + + private class StreamHolder { + + private final InputStream stream; + + private final String name; + + private StreamHolder(InputStream stream, String name) { + this.stream = stream; + this.name = name; + } + + public InputStream getStream() { + return stream; + } + + public String getName() { + return name; + } + + } + + +} diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/handler/FileTransferringMessageHandler.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/handler/FileTransferringMessageHandler.java index c266c1bbd7..edca07fbc3 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/handler/FileTransferringMessageHandler.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/handler/FileTransferringMessageHandler.java @@ -16,29 +16,15 @@ package org.springframework.integration.file.remote.handler; -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.BeanFactoryAware; import org.springframework.expression.Expression; import org.springframework.integration.Message; -import org.springframework.integration.MessageDeliveryException; -import org.springframework.integration.MessagingException; -import org.springframework.integration.file.DefaultFileNameGenerator; import org.springframework.integration.file.FileNameGenerator; -import org.springframework.integration.file.remote.RemoteFileUtils; -import org.springframework.integration.file.remote.session.Session; +import org.springframework.integration.file.remote.RemoteFileTemplate; import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.integration.handler.AbstractMessageHandler; -import org.springframework.integration.handler.ExpressionEvaluatingMessageProcessor; import org.springframework.util.Assert; -import org.springframework.util.StringUtils; /** * A {@link org.springframework.integration.core.MessageHandler} implementation that transfers files to a remote server. @@ -53,56 +39,32 @@ import org.springframework.util.StringUtils; */ public class FileTransferringMessageHandler extends AbstractMessageHandler { - private volatile String temporaryFileSuffix =".writing"; - - private final SessionFactory sessionFactory; - - private volatile boolean autoCreateDirectory = false; - - private volatile boolean useTemporaryFileName = true; - - private volatile ExpressionEvaluatingMessageProcessor directoryExpressionProcessor; - - private volatile ExpressionEvaluatingMessageProcessor temporaryDirectoryExpressionProcessor; - - private volatile FileNameGenerator fileNameGenerator = new DefaultFileNameGenerator(); - - private volatile boolean fileNameGeneratorSet; - - private volatile String charset = "UTF-8"; - - private volatile String remoteFileSeparator = "/"; - - private volatile boolean hasExplicitlySetSuffix; - + private final RemoteFileTemplate remoteFileTemplate; public FileTransferringMessageHandler(SessionFactory sessionFactory) { Assert.notNull(sessionFactory, "sessionFactory must not be null"); - this.sessionFactory = sessionFactory; + this.remoteFileTemplate = new RemoteFileTemplate(sessionFactory); } public void setAutoCreateDirectory(boolean autoCreateDirectory) { - this.autoCreateDirectory = autoCreateDirectory; + this.remoteFileTemplate.setAutoCreateDirectory(autoCreateDirectory); } public void setRemoteFileSeparator(String remoteFileSeparator) { - Assert.notNull(remoteFileSeparator, "'remoteFileSeparator' must not be null"); - this.remoteFileSeparator = remoteFileSeparator; + this.remoteFileTemplate.setRemoteFileSeparator(remoteFileSeparator); } public void setRemoteDirectoryExpression(Expression remoteDirectoryExpression) { - Assert.notNull(remoteDirectoryExpression, "remoteDirectoryExpression must not be null"); - this.directoryExpressionProcessor = new ExpressionEvaluatingMessageProcessor(remoteDirectoryExpression, String.class); + this.remoteFileTemplate.setRemoteDirectoryExpression(remoteDirectoryExpression); } public void setTemporaryRemoteDirectoryExpression(Expression temporaryRemoteDirectoryExpression) { - Assert.notNull(temporaryRemoteDirectoryExpression, "temporaryRemoteDirectoryExpression must not be null"); - this.temporaryDirectoryExpressionProcessor = new ExpressionEvaluatingMessageProcessor(temporaryRemoteDirectoryExpression, String.class); + this.remoteFileTemplate.setTemporaryRemoteDirectoryExpression(temporaryRemoteDirectoryExpression); } protected String getTemporaryFileSuffix() { - return this.temporaryFileSuffix; + return this.remoteFileTemplate.getTemporaryFileSuffix(); } /** @@ -113,198 +75,36 @@ public class FileTransferringMessageHandler extends AbstractMessageHandler { } protected boolean isUseTemporaryFileName() { - return useTemporaryFileName; + return this.remoteFileTemplate.isUseTemporaryFileName(); } public void setUseTemporaryFileName(boolean useTemporaryFileName) { - this.useTemporaryFileName = useTemporaryFileName; + this.remoteFileTemplate.setUseTemporaryFileName(useTemporaryFileName); } public void setFileNameGenerator(FileNameGenerator fileNameGenerator) { - this.fileNameGenerator = (fileNameGenerator != null) ? fileNameGenerator : new DefaultFileNameGenerator(); - this.fileNameGeneratorSet = fileNameGenerator != null; + this.remoteFileTemplate.setFileNameGenerator(fileNameGenerator); } public void setCharset(String charset) { - this.charset = charset; + this.remoteFileTemplate.setCharset(charset); } public void setTemporaryFileSuffix(String temporaryFileSuffix) { - Assert.notNull(temporaryFileSuffix, "'temporaryFileSuffix' must not be null"); - this.hasExplicitlySetSuffix = true; - this.temporaryFileSuffix = temporaryFileSuffix; + this.remoteFileTemplate.setTemporaryFileSuffix(temporaryFileSuffix); } @Override protected void onInit() throws Exception { - Assert.notNull(this.directoryExpressionProcessor, "remoteDirectoryExpression is required"); - BeanFactory beanFactory = this.getBeanFactory(); - if (beanFactory != null) { - this.directoryExpressionProcessor.setBeanFactory(beanFactory); - if (this.temporaryDirectoryExpressionProcessor != null) { - this.temporaryDirectoryExpressionProcessor.setBeanFactory(beanFactory); - } - if (!this.fileNameGeneratorSet && this.fileNameGenerator instanceof BeanFactoryAware) { - ((BeanFactoryAware) this.fileNameGenerator).setBeanFactory(beanFactory); - } - } - if (this.autoCreateDirectory){ - Assert.hasText(this.remoteFileSeparator, "'remoteFileSeparator' must not be empty when 'autoCreateDirectory' is set to 'true'"); - } - if (hasExplicitlySetSuffix && !useTemporaryFileName){ - this.logger.warn("Since 'use-temporary-file-name' is set to 'false' the value of 'temporary-file-suffix' has no effect"); - } + this.remoteFileTemplate.setBeanFactory(this.getBeanFactory()); + this.remoteFileTemplate.afterPropertiesSet(); } @Override protected void handleMessageInternal(Message message) throws Exception { - StreamHolder inputStreamHolder = this.payloadToInputStream(message); - if (inputStreamHolder != null) { - Session session = this.sessionFactory.getSession(); - String fileName = inputStreamHolder.getName(); - try { - String remoteDirectory = this.directoryExpressionProcessor.processMessage(message); - String temporaryRemoteDirectory = remoteDirectory; - if (this.temporaryDirectoryExpressionProcessor != null){ - temporaryRemoteDirectory = this.temporaryDirectoryExpressionProcessor.processMessage(message); - } - fileName = this.fileNameGenerator.generateFileName(message); - this.sendFileToRemoteDirectory(inputStreamHolder.getStream(), temporaryRemoteDirectory, remoteDirectory, fileName, session); - } - catch (FileNotFoundException e) { - throw new MessageDeliveryException(message, - "File [" + inputStreamHolder.getName() + "] not found in local working directory; it was moved or deleted unexpectedly.", e); - } - catch (IOException e) { - throw new MessageDeliveryException(message, - "Failed to transfer file [" + inputStreamHolder.getName() + " -> " + fileName + "] from local directory to remote directory.", e); - } - catch (Exception e) { - throw new MessageDeliveryException(message, - "Error handling message for file [" + inputStreamHolder.getName() + " -> " + fileName + "]", e); - } - finally { - if (session != null) { - session.close(); - } - } - } - else { - // A null holder means a File payload that does not exist. - if (logger.isWarnEnabled()) { - logger.warn("File " + message.getPayload() + " does not exist"); - } - } - } - - private StreamHolder payloadToInputStream(Message message) throws MessageDeliveryException { - try { - Object payload = message.getPayload(); - InputStream dataInputStream = null; - String name = null; - if (payload instanceof File) { - File inputFile = (File) payload; - if (inputFile.exists()) { - dataInputStream = new BufferedInputStream(new FileInputStream(inputFile)); - name = inputFile.getAbsolutePath(); - } - } - else if (payload instanceof byte[] || payload instanceof String) { - byte[] bytes = null; - if (payload instanceof String) { - bytes = ((String) payload).getBytes(this.charset); - name = "String payload"; - } - else { - bytes = (byte[]) payload; - name = "byte[] payload"; - } - dataInputStream = new ByteArrayInputStream(bytes); - } - else { - throw new IllegalArgumentException("Unsupported payload type. The only supported payloads are " + - "java.io.File, java.lang.String, and byte[]"); - } - if (dataInputStream == null) { - return null; - } - else { - return new StreamHolder(dataInputStream, name); - } - } - catch (Exception e) { - throw new MessageDeliveryException(message, "Failed to create sendable file.", e); - } - } - - private void sendFileToRemoteDirectory(InputStream inputStream, String temporaryRemoteDirectory, - String remoteDirectory, String fileName, Session session) throws FileNotFoundException, IOException { - - remoteDirectory = this.normalizeDirectoryPath(remoteDirectory); - temporaryRemoteDirectory = this.normalizeDirectoryPath(temporaryRemoteDirectory); - - String remoteFilePath = remoteDirectory + fileName; - String tempRemoteFilePath = temporaryRemoteDirectory + fileName; - // write remote file first with temporary file extension if enabled - - String tempFilePath = tempRemoteFilePath + (useTemporaryFileName ? this.temporaryFileSuffix : ""); - - if (this.autoCreateDirectory) { - try { - RemoteFileUtils.makeDirectories(remoteDirectory, session, this.remoteFileSeparator, this.logger); - } - catch (IllegalStateException e) { - // Revert to old FTP behavior if recursive mkdir fails, for backwards compatibility - session.mkdir(remoteDirectory); - } - } - - try { - session.write(inputStream, tempFilePath); - // then rename it to its final name if necessary - if (useTemporaryFileName){ - session.rename(tempFilePath, remoteFilePath); - } - } - catch (Exception e) { - throw new MessagingException("Failed to write to '" + tempFilePath + "' while uploading the file", e); - } - finally { - inputStream.close(); - } - } - - private String normalizeDirectoryPath(String directoryPath){ - if (!StringUtils.hasText(directoryPath)) { - directoryPath = ""; - } - else if (!directoryPath.endsWith(this.remoteFileSeparator)) { - directoryPath += this.remoteFileSeparator; - } - return directoryPath; - } - - private class StreamHolder { - - private final InputStream stream; - - private final String name; - - private StreamHolder(InputStream stream, String name) { - this.stream = stream; - this.name = name; - } - - public InputStream getStream() { - return stream; - } - - public String getName() { - return name; - } - + this.remoteFileTemplate.send(message); } } diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParserTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParserTests.java index d6b2f8e98a..c8980b86e0 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParserTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParserTests.java @@ -64,15 +64,15 @@ public class FtpOutboundChannelAdapterParserTests { assertEquals(channel, TestUtils.getPropertyValue(consumer, "inputChannel")); assertEquals("ftpOutbound", ((EventDrivenConsumer)consumer).getComponentName()); FileTransferringMessageHandler handler = TestUtils.getPropertyValue(consumer, "handler", FileTransferringMessageHandler.class); - String remoteFileSeparator = (String) TestUtils.getPropertyValue(handler, "remoteFileSeparator"); + String remoteFileSeparator = (String) TestUtils.getPropertyValue(handler, "remoteFileTemplate.remoteFileSeparator"); assertNotNull(remoteFileSeparator); - assertEquals(".foo", TestUtils.getPropertyValue(handler, "temporaryFileSuffix", String.class)); + assertEquals(".foo", TestUtils.getPropertyValue(handler, "remoteFileTemplate.temporaryFileSuffix", String.class)); assertEquals("", remoteFileSeparator); - assertEquals(ac.getBean("fileNameGenerator"), TestUtils.getPropertyValue(handler, "fileNameGenerator")); - assertEquals("UTF-8", TestUtils.getPropertyValue(handler, "charset")); - assertNotNull(TestUtils.getPropertyValue(handler, "directoryExpressionProcessor")); - assertNotNull(TestUtils.getPropertyValue(handler, "temporaryDirectoryExpressionProcessor")); - Object sfProperty = TestUtils.getPropertyValue(handler, "sessionFactory"); + assertEquals(ac.getBean("fileNameGenerator"), TestUtils.getPropertyValue(handler, "remoteFileTemplate.fileNameGenerator")); + assertEquals("UTF-8", TestUtils.getPropertyValue(handler, "remoteFileTemplate.charset")); + assertNotNull(TestUtils.getPropertyValue(handler, "remoteFileTemplate.directoryExpressionProcessor")); + assertNotNull(TestUtils.getPropertyValue(handler, "remoteFileTemplate.temporaryDirectoryExpressionProcessor")); + Object sfProperty = TestUtils.getPropertyValue(handler, "remoteFileTemplate.sessionFactory"); assertEquals(DefaultFtpSessionFactory.class, sfProperty.getClass()); DefaultFtpSessionFactory sessionFactory = (DefaultFtpSessionFactory) sfProperty; assertEquals("localhost", TestUtils.getPropertyValue(sessionFactory, "host")); @@ -99,7 +99,7 @@ public class FtpOutboundChannelAdapterParserTests { ApplicationContext ac = new ClassPathXmlApplicationContext( "FtpOutboundChannelAdapterParserTests-context.xml", this.getClass()); Object adapter = ac.getBean("simpleAdapter"); - Object sfProperty = TestUtils.getPropertyValue(adapter, "handler.sessionFactory"); + Object sfProperty = TestUtils.getPropertyValue(adapter, "handler.remoteFileTemplate.sessionFactory"); assertEquals(CachingSessionFactory.class, sfProperty.getClass()); Object innerSfProperty = TestUtils.getPropertyValue(sfProperty, "sessionFactory"); assertEquals(DefaultFtpSessionFactory.class, innerSfProperty.getClass()); @@ -121,7 +121,7 @@ public class FtpOutboundChannelAdapterParserTests { new ClassPathXmlApplicationContext("FtpOutboundChannelAdapterParserTests-context.xml", this.getClass()); FileTransferringMessageHandler handler = (FileTransferringMessageHandler)TestUtils.getPropertyValue(ac.getBean("ftpOutbound3"), "handler"); - assertFalse((Boolean)TestUtils.getPropertyValue(handler,"useTemporaryFileName")); + assertFalse((Boolean)TestUtils.getPropertyValue(handler,"remoteFileTemplate.useTemporaryFileName")); } @Test @@ -131,16 +131,16 @@ public class FtpOutboundChannelAdapterParserTests { Object consumer = ac.getBean("withBeanExpressions"); FileTransferringMessageHandler handler = TestUtils.getPropertyValue(consumer, "handler", FileTransferringMessageHandler.class); ExpressionEvaluatingMessageProcessor dirExpProc = TestUtils.getPropertyValue(handler, - "directoryExpressionProcessor", ExpressionEvaluatingMessageProcessor.class); + "remoteFileTemplate.directoryExpressionProcessor", ExpressionEvaluatingMessageProcessor.class); assertNotNull(dirExpProc); Message message = MessageBuilder.withPayload("qux").build(); assertEquals("foo", dirExpProc.processMessage(message)); ExpressionEvaluatingMessageProcessor tempDirExpProc = TestUtils.getPropertyValue(handler, - "temporaryDirectoryExpressionProcessor", ExpressionEvaluatingMessageProcessor.class); + "remoteFileTemplate.temporaryDirectoryExpressionProcessor", ExpressionEvaluatingMessageProcessor.class); assertNotNull(tempDirExpProc); assertEquals("bar", tempDirExpProc.processMessage(message)); DefaultFileNameGenerator generator = TestUtils.getPropertyValue(handler, - "fileNameGenerator", DefaultFileNameGenerator.class); + "remoteFileTemplate.fileNameGenerator", DefaultFileNameGenerator.class); assertNotNull(generator); assertEquals("baz", generator.generateFileName(message)); } diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpsOutboundChannelAdapterParserTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpsOutboundChannelAdapterParserTests.java index 7611b69cb9..cac33c6ce7 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpsOutboundChannelAdapterParserTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpsOutboundChannelAdapterParserTests.java @@ -44,9 +44,9 @@ public class FtpsOutboundChannelAdapterParserTests { assertEquals(ac.getBean("ftpChannel"), TestUtils.getPropertyValue(consumer, "inputChannel")); assertEquals("ftpOutbound", ((EventDrivenConsumer)consumer).getComponentName()); FileTransferringMessageHandler handler = TestUtils.getPropertyValue(consumer, "handler", FileTransferringMessageHandler.class); - assertEquals(ac.getBean("fileNameGenerator"), TestUtils.getPropertyValue(handler, "fileNameGenerator")); - assertEquals("UTF-8", TestUtils.getPropertyValue(handler, "charset")); - DefaultFtpsSessionFactory sf = TestUtils.getPropertyValue(handler, "sessionFactory", DefaultFtpsSessionFactory.class); + assertEquals(ac.getBean("fileNameGenerator"), TestUtils.getPropertyValue(handler, "remoteFileTemplate.fileNameGenerator")); + assertEquals("UTF-8", TestUtils.getPropertyValue(handler, "remoteFileTemplate.charset")); + DefaultFtpsSessionFactory sf = TestUtils.getPropertyValue(handler, "remoteFileTemplate.sessionFactory", DefaultFtpsSessionFactory.class); assertEquals("localhost", TestUtils.getPropertyValue(sf, "host")); assertEquals(22, TestUtils.getPropertyValue(sf, "port")); } diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpOutboundTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpOutboundTests.java index 9514d21cc1..43945ca803 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpOutboundTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpOutboundTests.java @@ -56,6 +56,7 @@ import org.springframework.integration.MessageChannel; import org.springframework.integration.core.PollableChannel; import org.springframework.integration.file.FileNameGenerator; import org.springframework.integration.file.remote.FileInfo; +import org.springframework.integration.file.remote.RemoteFileTemplate; import org.springframework.integration.file.remote.handler.FileTransferringMessageHandler; import org.springframework.integration.ftp.session.AbstractFtpSessionFactory; import org.springframework.integration.message.GenericMessage; @@ -94,6 +95,7 @@ public class FtpOutboundTests { FileTransferringMessageHandler handler = new FileTransferringMessageHandler(sessionFactory); handler.setRemoteDirectoryExpression(new LiteralExpression("remote-target-dir")); handler.setFileNameGenerator(new FileNameGenerator() { + @Override public String generateFileName(Message message) { return "handlerContent.test"; } @@ -117,6 +119,7 @@ public class FtpOutboundTests { FileTransferringMessageHandler handler = new FileTransferringMessageHandler(sessionFactory); handler.setRemoteDirectoryExpression(new LiteralExpression("remote-target-dir")); handler.setFileNameGenerator(new FileNameGenerator() { + @Override public String generateFileName(Message message) { return "handlerContent.test"; } @@ -138,6 +141,7 @@ public class FtpOutboundTests { FileTransferringMessageHandler handler = new FileTransferringMessageHandler(sessionFactory); handler.setRemoteDirectoryExpression(new LiteralExpression(targetDir.getName())); handler.setFileNameGenerator(new FileNameGenerator() { + @Override public String generateFileName(Message message) { return ((File)message.getPayload()).getName() + ".test"; } @@ -163,6 +167,7 @@ public class FtpOutboundTests { FileTransferringMessageHandler handler = new FileTransferringMessageHandler(sessionFactory); handler.setRemoteDirectoryExpression(new LiteralExpression(targetDir.getName())); handler.setFileNameGenerator(new FileNameGenerator() { + @Override public String generateFileName(Message message) { return ((File)message.getPayload()).getName() + ".test"; } @@ -172,7 +177,7 @@ public class FtpOutboundTests { File srcFile = new File(UUID.randomUUID() + ".txt"); - Log logger = spy(TestUtils.getPropertyValue(handler, "logger", Log.class)); + Log logger = spy(TestUtils.getPropertyValue(handler, "remoteFileTemplate.logger", Log.class)); when(logger.isWarnEnabled()).thenReturn(true); final AtomicReference logged = new AtomicReference(); doAnswer(new Answer(){ @@ -184,7 +189,8 @@ public class FtpOutboundTests { return null; } }).when(logger).warn(Mockito.anyString()); - new DirectFieldAccessor(handler).setPropertyValue("logger", logger); + RemoteFileTemplate template = TestUtils.getPropertyValue(handler, "remoteFileTemplate", RemoteFileTemplate.class); + new DirectFieldAccessor(template).setPropertyValue("logger", logger); handler.handleMessage(new GenericMessage(srcFile)); assertNotNull(logged.get()); assertEquals("File " + srcFile.toString() + " does not exist", logged.get()); @@ -242,6 +248,7 @@ public class FtpOutboundTests { when(ftpClient.changeWorkingDirectory(Mockito.anyString())).thenReturn(true); when(ftpClient.printWorkingDirectory()).thenReturn("remote-target-dir"); when(ftpClient.storeFile(Mockito.anyString(), Mockito.any(InputStream.class))).thenAnswer(new Answer() { + @Override public Boolean answer(InvocationOnMock invocation) throws Throwable { String fileName = (String) invocation.getArguments()[0]; InputStream fis = (InputStream) invocation.getArguments()[1]; @@ -250,6 +257,7 @@ public class FtpOutboundTests { } }); when(ftpClient.rename(Mockito.anyString(), Mockito.anyString())).thenAnswer(new Answer() { + @Override public Boolean answer(InvocationOnMock invocation) throws Throwable { File file = new File((String) invocation.getArguments()[0]); diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/OutboundChannelAdapterParserCachingTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/OutboundChannelAdapterParserCachingTests.java index 3b44ecf485..9f0ca17035 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/OutboundChannelAdapterParserCachingTests.java +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/OutboundChannelAdapterParserCachingTests.java @@ -44,13 +44,13 @@ public class OutboundChannelAdapterParserCachingTests { @Test public void cachingAdapter() { - Object sessionFactory = TestUtils.getPropertyValue(cachingAdapter, "handler.sessionFactory"); + Object sessionFactory = TestUtils.getPropertyValue(cachingAdapter, "handler.remoteFileTemplate.sessionFactory"); assertEquals(CachingSessionFactory.class, sessionFactory.getClass()); } @Test public void nonCachingAdapter() { - Object sessionFactory = TestUtils.getPropertyValue(nonCachingAdapter, "handler.sessionFactory"); + Object sessionFactory = TestUtils.getPropertyValue(nonCachingAdapter, "handler.remoteFileTemplate.sessionFactory"); assertEquals(DefaultSftpSessionFactory.class, sessionFactory.getClass()); } diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/OutboundChannelAdapterParserTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/OutboundChannelAdapterParserTests.java index c9c90a5bbd..8a1aa1ef4b 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/OutboundChannelAdapterParserTests.java +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/OutboundChannelAdapterParserTests.java @@ -66,17 +66,17 @@ public class OutboundChannelAdapterParserTests { assertEquals(channel, TestUtils.getPropertyValue(consumer, "inputChannel")); assertEquals("sftpOutboundAdapter", ((EventDrivenConsumer)consumer).getComponentName()); FileTransferringMessageHandler handler = TestUtils.getPropertyValue(consumer, "handler", FileTransferringMessageHandler.class); - String remoteFileSeparator = (String) TestUtils.getPropertyValue(handler, "remoteFileSeparator"); + String remoteFileSeparator = (String) TestUtils.getPropertyValue(handler, "remoteFileTemplate.remoteFileSeparator"); assertNotNull(remoteFileSeparator); assertEquals(".", remoteFileSeparator); - assertEquals(".bar", TestUtils.getPropertyValue(handler, "temporaryFileSuffix", String.class)); - Expression remoteDirectoryExpression = (Expression) TestUtils.getPropertyValue(handler, "directoryExpressionProcessor.expression"); + assertEquals(".bar", TestUtils.getPropertyValue(handler, "remoteFileTemplate.temporaryFileSuffix", String.class)); + Expression remoteDirectoryExpression = (Expression) TestUtils.getPropertyValue(handler, "remoteFileTemplate.directoryExpressionProcessor.expression"); assertNotNull(remoteDirectoryExpression); assertTrue(remoteDirectoryExpression instanceof LiteralExpression); - assertNotNull(TestUtils.getPropertyValue(handler, "temporaryDirectoryExpressionProcessor")); - assertEquals(context.getBean("fileNameGenerator"), TestUtils.getPropertyValue(handler, "fileNameGenerator")); - assertEquals("UTF-8", TestUtils.getPropertyValue(handler, "charset")); - CachingSessionFactory sessionFactory = TestUtils.getPropertyValue(handler, "sessionFactory", CachingSessionFactory.class); + assertNotNull(TestUtils.getPropertyValue(handler, "remoteFileTemplate.temporaryDirectoryExpressionProcessor")); + assertEquals(context.getBean("fileNameGenerator"), TestUtils.getPropertyValue(handler, "remoteFileTemplate.fileNameGenerator")); + assertEquals("UTF-8", TestUtils.getPropertyValue(handler, "remoteFileTemplate.charset")); + CachingSessionFactory sessionFactory = TestUtils.getPropertyValue(handler, "remoteFileTemplate.sessionFactory", CachingSessionFactory.class); DefaultSftpSessionFactory clientFactory = TestUtils.getPropertyValue(sessionFactory, "sessionFactory", DefaultSftpSessionFactory.class); assertEquals("localhost", TestUtils.getPropertyValue(clientFactory, "host")); assertEquals(2222, TestUtils.getPropertyValue(clientFactory, "port")); @@ -101,14 +101,14 @@ public class OutboundChannelAdapterParserTests { assertEquals(context.getBean("inputChannel"), TestUtils.getPropertyValue(consumer, "inputChannel")); assertEquals("sftpOutboundAdapterWithExpression", ((EventDrivenConsumer)consumer).getComponentName()); FileTransferringMessageHandler handler = TestUtils.getPropertyValue(consumer, "handler", FileTransferringMessageHandler.class); - SpelExpression remoteDirectoryExpression = (SpelExpression) TestUtils.getPropertyValue(handler, "directoryExpressionProcessor.expression"); + SpelExpression remoteDirectoryExpression = (SpelExpression) TestUtils.getPropertyValue(handler, "remoteFileTemplate.directoryExpressionProcessor.expression"); assertNotNull(remoteDirectoryExpression); assertEquals("'foo' + '/' + 'bar'", remoteDirectoryExpression.getExpressionString()); - FileNameGenerator generator = (FileNameGenerator) TestUtils.getPropertyValue(handler, "fileNameGenerator"); + FileNameGenerator generator = (FileNameGenerator) TestUtils.getPropertyValue(handler, "remoteFileTemplate.fileNameGenerator"); String fileNameGeneratorExpression = (String) TestUtils.getPropertyValue(generator, "expression"); assertEquals("payload.getName() + '-foo'", fileNameGeneratorExpression); - assertEquals("UTF-8", TestUtils.getPropertyValue(handler, "charset")); - assertNull(TestUtils.getPropertyValue(handler, "temporaryDirectoryExpressionProcessor")); + assertEquals("UTF-8", TestUtils.getPropertyValue(handler, "remoteFileTemplate.charset")); + assertNull(TestUtils.getPropertyValue(handler, "remoteFileTemplate.temporaryDirectoryExpressionProcessor")); } @@ -118,7 +118,7 @@ public class OutboundChannelAdapterParserTests { new ClassPathXmlApplicationContext("OutboundChannelAdapterParserTests-context.xml", this.getClass()); Object consumer = context.getBean("sftpOutboundAdapterWithNoTemporaryFileName"); FileTransferringMessageHandler handler = TestUtils.getPropertyValue(consumer, "handler", FileTransferringMessageHandler.class); - assertFalse((Boolean)TestUtils.getPropertyValue(handler,"useTemporaryFileName")); + assertFalse((Boolean)TestUtils.getPropertyValue(handler,"remoteFileTemplate.useTemporaryFileName")); } @Test From 1d0c28852fef7263182ad65faeb2dfa0174dfda0 Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Fri, 15 Nov 2013 15:13:07 +0200 Subject: [PATCH 03/29] INT-3204 RemoteFileTemplate Phase II Part 1 - AbstractInboundFileSynchronizer INT-3204 RemoteFileTemplate Phase II, Part 2 - AbstractRemoteFileOutboundGateway - Encapsulate Session.readRaw and .finalizeRaw in template.get() INT-3204 Polishing - PR Comments JIRA: https://jira.springsource.org/browse/INT-3204 INT-3204: Polishing --- .../file/remote/InputStreamCallback.java | 40 +++++ .../file/remote/RemoteFileOperations.java | 45 ++++- .../file/remote/RemoteFileTemplate.java | 158 ++++++++++++++---- .../file/remote/SessionCallback.java | 43 +++++ .../remote/SessionCallbackWithoutResult.java | 48 ++++++ .../AbstractRemoteFileOutboundGateway.java | 116 ++++++------- .../AbstractInboundFileSynchronizer.java | 59 +++---- .../FtpInboundChannelAdapterParserTests.java | 7 +- .../config/FtpOutboundGatewayParserTests.java | 9 +- .../ftp/outbound/FtpServerOutboundTests.java | 45 ++++- ...boundChannelAdapterParserCachingTests.java | 4 +- .../SftpOutboundGatewayParserTests.java | 9 +- 12 files changed, 446 insertions(+), 137 deletions(-) create mode 100644 spring-integration-file/src/main/java/org/springframework/integration/file/remote/InputStreamCallback.java create mode 100644 spring-integration-file/src/main/java/org/springframework/integration/file/remote/SessionCallback.java create mode 100644 spring-integration-file/src/main/java/org/springframework/integration/file/remote/SessionCallbackWithoutResult.java diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/InputStreamCallback.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/InputStreamCallback.java new file mode 100644 index 0000000000..47bf6284af --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/InputStreamCallback.java @@ -0,0 +1,40 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.file.remote; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Callback for stream-based file retrieval using a RemoteFileOperations. + * + * @author Gary Russell + * @since 3.0 + * + */ +public interface InputStreamCallback { + + /** + * Called with the InputStream for the remote file. The caller will + * take care of closing the stream and finalizing the file retrieval operation after + * this method exits. + * + * @param stream The InputStream. + * @throws IOException + */ + void doWithInputStream(InputStream stream) throws IOException; + +} diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java index 1b61d9cb23..3f5ca23732 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java @@ -18,12 +18,53 @@ package org.springframework.integration.file.remote; import org.springframework.integration.Message; /** + * Strategy for performing operations on remote files. + * * @author Gary Russell * @since 3.0 * */ -public interface RemoteFileOperations { +public interface RemoteFileOperations { - public abstract void send(Message message) throws Exception; + /** + * Send a file to a remote server, based on information in a message. + * + * @param message The message + * @throws Exception + */ + void send(Message message); + + /** + * Retrieve a remote file as an InputStream, based on information in a message. + * + * @param callback the callback. + * @return true if the operation was successful. + */ + boolean get(Message message, InputStreamCallback callback); + + /** + * Remove a remote file. + * + * @param path The full path to the file. + * @return true when successful + */ + boolean remove(String path); + + /** + * Rename a remote file, creating directories if needed. + * + * @param fromPath The current path. + * @param toPath The new path. + */ + void rename(String fromPath, String toPath); + + /** + * Execute the callback's doInSession method after obtaining a session. + * Reliably closes the session when the method exits. + * + * @param callback the SessionCallback. + * @return The result of the callback method. + */ + T execute(SessionCallback callback); } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java index f96fe3697d..02e3918ee2 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java @@ -52,10 +52,13 @@ import org.springframework.util.StringUtils; * @since 3.0 * */ -public class RemoteFileTemplate implements RemoteFileOperations, InitializingBean, BeanFactoryAware { +public class RemoteFileTemplate implements RemoteFileOperations, InitializingBean, BeanFactoryAware { private final Log logger = LogFactory.getLog(this.getClass()); + /** + * the {@link SessionFactory} for acquiring remote file Sessions. + */ private final SessionFactory sessionFactory; private volatile String temporaryFileSuffix =".writing"; @@ -68,6 +71,8 @@ public class RemoteFileTemplate implements RemoteFileOperations, Initializing private volatile ExpressionEvaluatingMessageProcessor temporaryDirectoryExpressionProcessor; + private volatile ExpressionEvaluatingMessageProcessor fileNameProcessor; + private volatile FileNameGenerator fileNameGenerator = new DefaultFileNameGenerator(); private volatile boolean fileNameGeneratorSet; @@ -104,6 +109,11 @@ public class RemoteFileTemplate implements RemoteFileOperations, Initializing this.temporaryDirectoryExpressionProcessor = new ExpressionEvaluatingMessageProcessor(temporaryRemoteDirectoryExpression, String.class); } + public void setFileNameExpression(Expression fileNameExpression) { + Assert.notNull(fileNameExpression, "fileNameExpression must not be null"); + this.fileNameProcessor = new ExpressionEvaluatingMessageProcessor(fileNameExpression, String.class); + } + public String getTemporaryFileSuffix() { return this.temporaryFileSuffix; } @@ -139,16 +149,20 @@ public class RemoteFileTemplate implements RemoteFileOperations, Initializing @Override public void afterPropertiesSet() throws Exception { - Assert.notNull(this.directoryExpressionProcessor, "remoteDirectoryExpression is required"); BeanFactory beanFactory = this.beanFactory; if (beanFactory != null) { - this.directoryExpressionProcessor.setBeanFactory(beanFactory); + if (this.directoryExpressionProcessor != null) { + this.directoryExpressionProcessor.setBeanFactory(beanFactory); + } if (this.temporaryDirectoryExpressionProcessor != null) { this.temporaryDirectoryExpressionProcessor.setBeanFactory(beanFactory); } if (!this.fileNameGeneratorSet && this.fileNameGenerator instanceof BeanFactoryAware) { ((BeanFactoryAware) this.fileNameGenerator).setBeanFactory(beanFactory); } + if (this.fileNameProcessor != null) { + this.fileNameProcessor.setBeanFactory(beanFactory); + } } if (this.autoCreateDirectory){ Assert.hasText(this.remoteFileSeparator, "'remoteFileSeparator' must not be empty when 'autoCreateDirectory' is set to 'true'"); @@ -159,37 +173,42 @@ public class RemoteFileTemplate implements RemoteFileOperations, Initializing } @Override - public void send(Message message) throws Exception { - StreamHolder inputStreamHolder = this.payloadToInputStream(message); + public void send(final Message message) { + Assert.notNull(this.directoryExpressionProcessor, "'remoteDirectoryExpression' is required"); + final StreamHolder inputStreamHolder = this.payloadToInputStream(message); if (inputStreamHolder != null) { - Session session = this.sessionFactory.getSession(); - String fileName = inputStreamHolder.getName(); - try { - String remoteDirectory = this.directoryExpressionProcessor.processMessage(message); - String temporaryRemoteDirectory = remoteDirectory; - if (this.temporaryDirectoryExpressionProcessor != null){ - temporaryRemoteDirectory = this.temporaryDirectoryExpressionProcessor.processMessage(message); + this.execute(new SessionCallbackWithoutResult() { + + @Override + public void doInSessionWithoutResult(Session session) throws IOException { + String fileName = inputStreamHolder.getName(); + try { + String remoteDirectory = RemoteFileTemplate.this.directoryExpressionProcessor + .processMessage(message); + String temporaryRemoteDirectory = remoteDirectory; + if (RemoteFileTemplate.this.temporaryDirectoryExpressionProcessor != null) { + temporaryRemoteDirectory = RemoteFileTemplate.this.temporaryDirectoryExpressionProcessor + .processMessage(message); + } + fileName = RemoteFileTemplate.this.fileNameGenerator.generateFileName(message); + RemoteFileTemplate.this.sendFileToRemoteDirectory(inputStreamHolder.getStream(), + temporaryRemoteDirectory, remoteDirectory, fileName, session); + } + catch (FileNotFoundException e) { + throw new MessageDeliveryException(message, "File [" + inputStreamHolder.getName() + + "] not found in local working directory; it was moved or deleted unexpectedly.", e); + } + catch (IOException e) { + throw new MessageDeliveryException(message, "Failed to transfer file [" + + inputStreamHolder.getName() + " -> " + fileName + + "] from local directory to remote directory.", e); + } + catch (Exception e) { + throw new MessageDeliveryException(message, "Error handling message for file [" + + inputStreamHolder.getName() + " -> " + fileName + "]", e); + } } - fileName = this.fileNameGenerator.generateFileName(message); - this.sendFileToRemoteDirectory(inputStreamHolder.getStream(), temporaryRemoteDirectory, remoteDirectory, fileName, session); - } - catch (FileNotFoundException e) { - throw new MessageDeliveryException(message, - "File [" + inputStreamHolder.getName() + "] not found in local working directory; it was moved or deleted unexpectedly.", e); - } - catch (IOException e) { - throw new MessageDeliveryException(message, - "Failed to transfer file [" + inputStreamHolder.getName() + " -> " + fileName + "] from local directory to remote directory.", e); - } - catch (Exception e) { - throw new MessageDeliveryException(message, - "Error handling message for file [" + inputStreamHolder.getName() + " -> " + fileName + "]", e); - } - finally { - if (session != null) { - session.close(); - } - } + }); } else { // A null holder means a File payload that does not exist. @@ -199,6 +218,79 @@ public class RemoteFileTemplate implements RemoteFileOperations, Initializing } } + @Override + public boolean remove(final String path) { + return this.execute(new SessionCallback() { + + @Override + public Boolean doInSession(Session session) throws IOException { + return session.remove(path); + } + }); + } + + @Override + public void rename(final String fromPath, final String toPath) { + Assert.hasText(fromPath, "Old filename cannot be null or empty"); + Assert.hasText(toPath, "New filename cannot be null or empty"); + + this.execute(new SessionCallbackWithoutResult() { + + @Override + public void doInSessionWithoutResult(Session session) throws IOException { + int lastSeparator = toPath.lastIndexOf(RemoteFileTemplate.this.remoteFileSeparator); + if (lastSeparator > 0) { + String remoteFileDirectory = toPath.substring(0, lastSeparator + 1); + RemoteFileUtils.makeDirectories(remoteFileDirectory, session, + RemoteFileTemplate.this.remoteFileSeparator, RemoteFileTemplate.this.logger); + } + session.rename(fromPath, toPath); + } + }); + } + + + @Override + public boolean get(final Message message, final InputStreamCallback callback) { + Assert.notNull(this.fileNameProcessor, "'fileNameProcessor' needed to use get"); + return this.execute(new SessionCallback() { + + @Override + public Boolean doInSession(Session session) throws IOException { + final String remotePath = RemoteFileTemplate.this.fileNameProcessor.processMessage(message); + InputStream inputStream = session.readRaw(remotePath); + callback.doWithInputStream(inputStream); + inputStream.close(); + return session.finalizeRaw(); + } + }); + } + + @Override + public T execute(SessionCallback callback) { + Session session = null; + try { + session = this.sessionFactory.getSession(); + Assert.notNull(session, "failed to acquire a Session"); + return callback.doInSession(session); + } + catch (IOException e) { + throw new MessagingException("Failed to execute on session", e); + } + finally { + if (session != null) { + try { + session.close(); + } + catch (Exception ignored) { + if (logger.isDebugEnabled()) { + logger.debug("failed to close Session", ignored); + } + } + } + } + } + private StreamHolder payloadToInputStream(Message message) throws MessageDeliveryException { try { Object payload = message.getPayload(); @@ -240,7 +332,7 @@ public class RemoteFileTemplate implements RemoteFileOperations, Initializing } private void sendFileToRemoteDirectory(InputStream inputStream, String temporaryRemoteDirectory, - String remoteDirectory, String fileName, Session session) throws FileNotFoundException, IOException { + String remoteDirectory, String fileName, Session session) throws IOException { remoteDirectory = this.normalizeDirectoryPath(remoteDirectory); temporaryRemoteDirectory = this.normalizeDirectoryPath(temporaryRemoteDirectory); diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/SessionCallback.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/SessionCallback.java new file mode 100644 index 0000000000..2f0fc06277 --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/SessionCallback.java @@ -0,0 +1,43 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.file.remote; + +import java.io.IOException; + +import org.springframework.integration.file.remote.session.Session; + +/** + * Callback invoked by {@code RemoteFileOperations.execute()) - allows multiple operations + * on a session. + * + * @author Gary Russell + * @since 3.0 + * + */ +public interface SessionCallback { + + /** + * Called within the context of a session. + * Perform some operation(s) on the session. The caller will take + * care of closing the session after this method exits. + * + * @param session The session. + * @return The result of type T. + * @throws IOException + */ + T doInSession(Session session) throws IOException; + +} diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/SessionCallbackWithoutResult.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/SessionCallbackWithoutResult.java new file mode 100644 index 0000000000..b13af2c978 --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/SessionCallbackWithoutResult.java @@ -0,0 +1,48 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.file.remote; + +import java.io.IOException; + +import org.springframework.integration.file.remote.session.Session; + +/** + * Simple convenience implementation of {@link SessionCallback} for cases where + * no result is returned. + * + * @author Gary Russell + * @since 3.0 + * + */ +public abstract class SessionCallbackWithoutResult implements SessionCallback { + + @Override + public Object doInSession(Session session) throws IOException { + this.doInSessionWithoutResult(session); + return null; + } + + /** + * Called within the context of a session. + * Perform some operation(s) on the session. The caller will take + * care of closing the session after this method exits. + * + * @param session The session. + * @throws IOException + */ + protected abstract void doInSessionWithoutResult(Session session) throws IOException; + +} diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java index bf4fd97df2..e1ece85a17 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java @@ -40,7 +40,8 @@ import org.springframework.integration.expression.ExpressionUtils; import org.springframework.integration.file.FileHeaders; import org.springframework.integration.file.filters.FileListFilter; import org.springframework.integration.file.remote.AbstractFileInfo; -import org.springframework.integration.file.remote.RemoteFileUtils; +import org.springframework.integration.file.remote.RemoteFileTemplate; +import org.springframework.integration.file.remote.SessionCallback; import org.springframework.integration.file.remote.session.Session; import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.integration.handler.AbstractReplyProducingMessageHandler; @@ -59,7 +60,7 @@ import org.springframework.util.StringUtils; */ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReplyProducingMessageHandler { - protected final SessionFactory sessionFactory; + private final RemoteFileTemplate remoteFileTemplate; protected final Command command; @@ -205,7 +206,8 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply public AbstractRemoteFileOutboundGateway(SessionFactory sessionFactory, String command, String expression) { - this.sessionFactory = sessionFactory; + Assert.notNull(sessionFactory, "'sessionFactory' cannot be null"); + this.remoteFileTemplate = new RemoteFileTemplate(sessionFactory); this.command = Command.toCommand(command); this.fileNameProcessor = new ExpressionEvaluatingMessageProcessor( new SpelExpressionParser().parseExpression(expression)); @@ -213,7 +215,8 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply public AbstractRemoteFileOutboundGateway(SessionFactory sessionFactory, Command command, String expression) { - this.sessionFactory = sessionFactory; + Assert.notNull(sessionFactory, "'sessionFactory' cannot be null"); + this.remoteFileTemplate = new RemoteFileTemplate(sessionFactory); this.command = command; this.fileNameProcessor = new ExpressionEvaluatingMessageProcessor( new SpelExpressionParser().parseExpression(expression)); @@ -330,88 +333,101 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply if (this.getBeanFactory() != null) { this.fileNameProcessor.setBeanFactory(this.getBeanFactory()); this.renameProcessor.setBeanFactory(this.getBeanFactory()); + this.remoteFileTemplate.setBeanFactory(this.getBeanFactory()); } } @Override protected Object handleRequestMessage(Message requestMessage) { - Session session = this.sessionFactory.getSession(); - try { - switch (this.command) { - case LS: - return doLs(requestMessage, session); - case GET: - return doGet(requestMessage, session); - case MGET: - return doMget(requestMessage, session); - case RM: - return doRm(requestMessage, session); - case MV: - return doMv(requestMessage, session); - default: - return null; - } - } - catch (IOException e) { - throw new MessagingException(requestMessage, e); - } - finally { - session.close(); + switch (this.command) { + case LS: + return doLs(requestMessage); + case GET: + return doGet(requestMessage); + case MGET: + return doMget(requestMessage); + case RM: + return doRm(requestMessage); + case MV: + return doMv(requestMessage); + default: + return null; } } - private Object doLs(Message requestMessage, Session session) throws IOException { + private Object doLs(Message requestMessage) { String dir = this.fileNameProcessor.processMessage(requestMessage); if (!dir.endsWith(this.remoteFileSeparator)) { dir += this.remoteFileSeparator; } - List payload = ls(session, dir); + final String fullDir = dir; + List payload = this.remoteFileTemplate.execute(new SessionCallback>() { + + @Override + public List doInSession(Session session) throws IOException { + return AbstractRemoteFileOutboundGateway.this.ls(session, fullDir); + } + }); return MessageBuilder.withPayload(payload) .setHeader(FileHeaders.REMOTE_DIRECTORY, dir) .build(); } - private Object doGet(Message requestMessage, Session session) throws IOException { - String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); - String remoteFilename = this.getRemoteFilename(remoteFilePath); - String remoteDir = this.getRemoteDirectory(remoteFilePath, remoteFilename); - File payload = this.get(requestMessage, session, remoteDir, remoteFilePath, remoteFilename, true); + private Object doGet(final Message requestMessage) { + final String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); + final String remoteFilename = this.getRemoteFilename(remoteFilePath); + final String remoteDir = this.getRemoteDirectory(remoteFilePath, remoteFilename); + File payload = this.remoteFileTemplate.execute(new SessionCallback() { + + @Override + public File doInSession(Session session) throws IOException { + return AbstractRemoteFileOutboundGateway.this.get(requestMessage, session, remoteDir, remoteFilePath, + remoteFilename, true); + + } + }); return MessageBuilder.withPayload(payload) .setHeader(FileHeaders.REMOTE_DIRECTORY, remoteDir) .setHeader(FileHeaders.REMOTE_FILE, remoteFilename) .build(); } - private Object doMget(Message requestMessage, Session session) throws IOException { - String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); - String remoteFilename = this.getRemoteFilename(remoteFilePath); - String remoteDir = this.getRemoteDirectory(remoteFilePath, remoteFilename); - List payload = this.mGet(requestMessage, session, remoteDir, remoteFilename); + private Object doMget(final Message requestMessage) { + final String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); + final String remoteFilename = this.getRemoteFilename(remoteFilePath); + final String remoteDir = this.getRemoteDirectory(remoteFilePath, remoteFilename); + List payload = this.remoteFileTemplate.execute(new SessionCallback>() { + + @Override + public List doInSession(Session session) throws IOException { + return AbstractRemoteFileOutboundGateway.this.mGet(requestMessage, session, remoteDir, remoteFilename); + } + }); return MessageBuilder.withPayload(payload) .setHeader(FileHeaders.REMOTE_DIRECTORY, remoteDir) .setHeader(FileHeaders.REMOTE_FILE, remoteFilename) .build(); } - private Object doRm(Message requestMessage, Session session) throws IOException { - String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); + private Object doRm(Message requestMessage) { + final String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); String remoteFilename = this.getRemoteFilename(remoteFilePath); String remoteDir = this.getRemoteDirectory(remoteFilePath, remoteFilename); - boolean payload = this.rm(session, remoteFilePath); + boolean payload = this.remoteFileTemplate.remove(remoteFilePath); return MessageBuilder.withPayload(payload) .setHeader(FileHeaders.REMOTE_DIRECTORY, remoteDir) .setHeader(FileHeaders.REMOTE_FILE, remoteFilename) .build(); } - private Object doMv(Message requestMessage, Session session) throws IOException { + private Object doMv(Message requestMessage) { String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); String remoteFilename = this.getRemoteFilename(remoteFilePath); String remoteDir = this.getRemoteDirectory(remoteFilePath, remoteFilename); String remoteFileNewPath = this.renameProcessor.processMessage(requestMessage); Assert.hasLength(remoteFileNewPath, "New filename cannot be empty"); - this.mv(session, remoteFilePath, remoteFileNewPath); + this.remoteFileTemplate.rename(remoteFilePath, remoteFileNewPath); return MessageBuilder.withPayload(Boolean.TRUE) .setHeader(FileHeaders.REMOTE_DIRECTORY, remoteDir) .setHeader(FileHeaders.REMOTE_FILE, remoteFilename) @@ -660,20 +676,6 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply return remoteFileName; } - protected boolean rm(Session session, String remoteFilePath) - throws IOException { - return session.remove(remoteFilePath); - } - - protected void mv(Session session, String remoteFilePath, String remoteFileNewPath) throws IOException { - int lastSeparator = remoteFileNewPath.lastIndexOf(this.remoteFileSeparator); - if (lastSeparator > 0) { - String remoteFileDirectory = remoteFileNewPath.substring(0, lastSeparator + 1); - RemoteFileUtils.makeDirectories(remoteFileDirectory, session, this.remoteFileSeparator, this.logger); - } - session.rename(remoteFilePath, remoteFileNewPath); - } - private File generateLocalDirectory(Message message, String remoteDirectory) { EvaluationContext evaluationContext = ExpressionUtils.createStandardEvaluationContext(this.getBeanFactory()); evaluationContext.setVariable("remoteDirectory", remoteDirectory); diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java index ba725a1283..e89d2a131a 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java @@ -34,6 +34,8 @@ import org.springframework.expression.Expression; import org.springframework.integration.MessagingException; import org.springframework.integration.expression.IntegrationEvaluationContextAware; import org.springframework.integration.file.filters.FileListFilter; +import org.springframework.integration.file.remote.RemoteFileTemplate; +import org.springframework.integration.file.remote.SessionCallback; import org.springframework.integration.file.remote.session.Session; import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.util.Assert; @@ -59,6 +61,8 @@ public abstract class AbstractInboundFileSynchronizer implements InboundFileS protected final Log logger = LogFactory.getLog(this.getClass()); + private final RemoteFileTemplate remoteFileTemplate; + private volatile EvaluationContext evaluationContext; private volatile String remoteFileSeparator = "/"; @@ -75,11 +79,6 @@ public abstract class AbstractInboundFileSynchronizer implements InboundFileS */ private volatile String remoteDirectory; - /** - * the {@link SessionFactory} for acquiring remote file Sessions. - */ - private final SessionFactory sessionFactory; - /** * An {@link FileListFilter} that runs against the remote file system view. */ @@ -102,7 +101,7 @@ public abstract class AbstractInboundFileSynchronizer implements InboundFileS */ public AbstractInboundFileSynchronizer(SessionFactory sessionFactory) { Assert.notNull(sessionFactory, "sessionFactory must not be null"); - this.sessionFactory = sessionFactory; + this.remoteFileTemplate = new RemoteFileTemplate(sessionFactory); } @@ -144,6 +143,7 @@ public abstract class AbstractInboundFileSynchronizer implements InboundFileS this.evaluationContext = evaluationContext; } + @Override public final void afterPropertiesSet() { Assert.notNull(this.remoteDirectory, "remoteDirectory must not be null"); Assert.notNull(this.evaluationContext, "evaluationContext must not be null"); @@ -157,36 +157,37 @@ public abstract class AbstractInboundFileSynchronizer implements InboundFileS return temporaryFileSuffix; } - public void synchronizeToLocalDirectory(File localDirectory) { - Session session = null; + @Override + public void synchronizeToLocalDirectory(final File localDirectory) { try { - session = this.sessionFactory.getSession(); - Assert.notNull(session, "failed to acquire a Session"); - F[] files = session.list(this.remoteDirectory); - if (!ObjectUtils.isEmpty(files)) { - Collection filteredFiles = this.filterFiles(files); - for (F file : filteredFiles) { - if (file != null) { - this.copyFileToLocalDirectory(this.remoteDirectory, file, localDirectory, session); + int transferred = this.remoteFileTemplate.execute(new SessionCallback() { + + @Override + public Integer doInSession(Session session) throws IOException { + F[] files = session.list(AbstractInboundFileSynchronizer.this.remoteDirectory); + if (!ObjectUtils.isEmpty(files)) { + Collection filteredFiles = AbstractInboundFileSynchronizer.this.filterFiles(files); + for (F file : filteredFiles) { + if (file != null) { + AbstractInboundFileSynchronizer.this.copyFileToLocalDirectory( + AbstractInboundFileSynchronizer.this.remoteDirectory, file, localDirectory, + session); + } + } + return filteredFiles.size(); + } + else { + return 0; } } + }); + if (logger.isDebugEnabled()) { + logger.debug(transferred + " files transferred"); } } - catch (IOException e) { + catch (Exception e) { throw new MessagingException("Problem occurred while synchronizing remote to local directory", e); } - finally { - if (session != null) { - try { - session.close(); - } - catch (Exception ignored) { - if (logger.isDebugEnabled()) { - logger.debug("failed to close Session", ignored); - } - } - } - } } private void copyFileToLocalDirectory(String remoteDirectoryPath, F remoteFile, File localDirectory, Session session) throws IOException { diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests.java index 4d40e37900..e3340a1289 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests.java @@ -84,7 +84,7 @@ public class FtpInboundChannelAdapterParserTests { assertEquals("", remoteFileSeparator); FtpSimplePatternFileListFilter filter = (FtpSimplePatternFileListFilter) TestUtils.getPropertyValue(fisync, "filter"); assertNotNull(filter); - Object sessionFactory = TestUtils.getPropertyValue(fisync, "sessionFactory"); + Object sessionFactory = TestUtils.getPropertyValue(fisync, "remoteFileTemplate.sessionFactory"); assertTrue(DefaultFtpSessionFactory.class.isAssignableFrom(sessionFactory.getClass())); FileListFilter acceptAllFilter = ac.getBean("acceptAllFilter", FileListFilter.class); assertTrue(TestUtils.getPropertyValue(inbound, "fileSource.scanner.filter.fileFilters", Collection.class).contains(acceptAllFilter)); @@ -107,7 +107,7 @@ public class FtpInboundChannelAdapterParserTests { ApplicationContext ac = new ClassPathXmlApplicationContext( "FtpInboundChannelAdapterParserTests-context.xml", this.getClass()); SourcePollingChannelAdapter adapter = ac.getBean("simpleAdapterWithCachedSessions", SourcePollingChannelAdapter.class); - Object sessionFactory = TestUtils.getPropertyValue(adapter, "source.synchronizer.sessionFactory"); + Object sessionFactory = TestUtils.getPropertyValue(adapter, "source.synchronizer.remoteFileTemplate.sessionFactory"); assertEquals(CachingSessionFactory.class, sessionFactory.getClass()); FtpInboundFileSynchronizer fisync = TestUtils.getPropertyValue(adapter, "source.synchronizer", FtpInboundFileSynchronizer.class); @@ -142,6 +142,7 @@ public class FtpInboundChannelAdapterParserTests { public static class TestSessionFactoryBean implements FactoryBean { + @Override @SuppressWarnings({ "rawtypes", "unchecked" }) public DefaultFtpSessionFactory getObject() throws Exception { DefaultFtpSessionFactory factory = mock(DefaultFtpSessionFactory.class); @@ -150,10 +151,12 @@ public class FtpInboundChannelAdapterParserTests { return factory; } + @Override public Class getObjectType() { return DefaultFtpSessionFactory.class; } + @Override public boolean isSingleton() { return true; } diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java index 4c28c196f4..b691b40e8f 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java @@ -20,7 +20,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import java.io.File; import java.lang.reflect.Method; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -75,7 +74,7 @@ public class FtpOutboundGatewayParserTests { FtpOutboundGateway gateway = TestUtils.getPropertyValue(gateway1, "handler", FtpOutboundGateway.class); assertEquals("X", TestUtils.getPropertyValue(gateway, "remoteFileSeparator")); - assertNotNull(TestUtils.getPropertyValue(gateway, "sessionFactory")); + assertNotNull(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory")); assertNotNull(TestUtils.getPropertyValue(gateway, "outputChannel")); assertEquals("local-test-dir", TestUtils.getPropertyValue(gateway, "localDirectoryExpression.literalValue")); assertFalse((Boolean) TestUtils.getPropertyValue(gateway, "autoCreateLocalDirectory")); @@ -97,8 +96,8 @@ public class FtpOutboundGatewayParserTests { FtpOutboundGateway gateway = TestUtils.getPropertyValue(gateway2, "handler", FtpOutboundGateway.class); assertEquals("X", TestUtils.getPropertyValue(gateway, "remoteFileSeparator")); - assertNotNull(TestUtils.getPropertyValue(gateway, "sessionFactory")); - assertTrue(TestUtils.getPropertyValue(gateway, "sessionFactory") instanceof CachingSessionFactory); + assertNotNull(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory")); + assertTrue(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory") instanceof CachingSessionFactory); assertNotNull(TestUtils.getPropertyValue(gateway, "outputChannel")); assertEquals("local-test-dir", TestUtils.getPropertyValue(gateway, "localDirectoryExpression.literalValue")); assertFalse((Boolean) TestUtils.getPropertyValue(gateway, "autoCreateLocalDirectory")); @@ -130,7 +129,7 @@ public class FtpOutboundGatewayParserTests { public void testGatewayMv() { FtpOutboundGateway gateway = TestUtils.getPropertyValue(gateway3, "handler", FtpOutboundGateway.class); - assertNotNull(TestUtils.getPropertyValue(gateway, "sessionFactory")); + assertNotNull(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory")); assertNotNull(TestUtils.getPropertyValue(gateway, "outputChannel")); assertEquals(Command.MV, TestUtils.getPropertyValue(gateway, "command")); assertEquals("'foo'", TestUtils.getPropertyValue(gateway, "renameProcessor.expression.expression")); diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java index 04d1f849e0..aa2ea8ace9 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java @@ -21,21 +21,30 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.util.List; +import org.apache.commons.net.ftp.FTPFile; import org.hamcrest.Matchers; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.springframework.beans.factory.BeanFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.expression.spel.standard.SpelExpressionParser; import org.springframework.integration.Message; import org.springframework.integration.channel.DirectChannel; import org.springframework.integration.core.PollableChannel; +import org.springframework.integration.file.remote.InputStreamCallback; +import org.springframework.integration.file.remote.RemoteFileTemplate; import org.springframework.integration.file.remote.session.Session; +import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.integration.ftp.TesFtpServer; import org.springframework.integration.message.GenericMessage; import org.springframework.test.context.ContextConfiguration; @@ -44,6 +53,8 @@ import org.springframework.util.FileCopyUtils; /** * @author Artem Bilan + * @author Gary Russell + * * @since 3.0 */ @ContextConfiguration @@ -51,7 +62,10 @@ import org.springframework.util.FileCopyUtils; public class FtpServerOutboundTests { @Autowired - public TesFtpServer ftpServer; + private TesFtpServer ftpServer; + + @Autowired + private SessionFactory ftpSessionFactory; @Autowired private PollableChannel output; @@ -176,7 +190,7 @@ public class FtpServerOutboundTests { @Test public void testInt3100RawGET() throws Exception { - Session session = this.ftpServer.ftpSessionFactory().getSession(); + Session session = this.ftpSessionFactory.getSession(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); FileCopyUtils.copy(session.readRaw("ftpSource/ftpSource1.txt"), baos); assertTrue(session.finalizeRaw()); @@ -190,5 +204,32 @@ public class FtpServerOutboundTests { session.close(); } + @Test + public void testRawGETWithTemplate() throws Exception { + RemoteFileTemplate template = new RemoteFileTemplate(this.ftpSessionFactory); + template.setFileNameExpression(new SpelExpressionParser().parseExpression("payload")); + template.setBeanFactory(mock(BeanFactory.class)); + template.afterPropertiesSet(); + final ByteArrayOutputStream baos1 = new ByteArrayOutputStream(); + assertTrue(template.get(new GenericMessage("ftpSource/ftpSource1.txt"), new InputStreamCallback() { + + @Override + public void doWithInputStream(InputStream stream) throws IOException { + FileCopyUtils.copy(stream, baos1); + } + })); + assertEquals("source1", new String(baos1.toByteArray())); + + final ByteArrayOutputStream baos2 = new ByteArrayOutputStream(); + assertTrue(template.get(new GenericMessage("ftpSource/ftpSource2.txt"), new InputStreamCallback() { + + @Override + public void doWithInputStream(InputStream stream) throws IOException { + FileCopyUtils.copy(stream, baos2); + } + })); + assertEquals("source2", new String(baos2.toByteArray())); + } + } diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserCachingTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserCachingTests.java index 57c2a659f3..b432292dd0 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserCachingTests.java +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserCachingTests.java @@ -46,7 +46,7 @@ public class InboundChannelAdapterParserCachingTests { @Test public void cachingAdapter() { - Object sessionFactory = TestUtils.getPropertyValue(cachingAdapter, "source.synchronizer.sessionFactory"); + Object sessionFactory = TestUtils.getPropertyValue(cachingAdapter, "source.synchronizer.remoteFileTemplate.sessionFactory"); assertEquals(CachingSessionFactory.class, sessionFactory.getClass()); Properties sessionConfig = TestUtils.getPropertyValue(sessionFactory, "sessionFactory.sessionConfig", Properties.class); assertNotNull(sessionConfig); @@ -55,7 +55,7 @@ public class InboundChannelAdapterParserCachingTests { @Test public void nonCachingAdapter() { - Object sessionFactory = TestUtils.getPropertyValue(nonCachingAdapter, "source.synchronizer.sessionFactory"); + Object sessionFactory = TestUtils.getPropertyValue(nonCachingAdapter, "source.synchronizer.remoteFileTemplate.sessionFactory"); assertEquals(DefaultSftpSessionFactory.class, sessionFactory.getClass()); } diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpOutboundGatewayParserTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpOutboundGatewayParserTests.java index c1b69f1712..8604639969 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpOutboundGatewayParserTests.java +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpOutboundGatewayParserTests.java @@ -20,7 +20,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import java.io.File; import java.lang.reflect.Method; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -73,7 +72,7 @@ public class SftpOutboundGatewayParserTests { SftpOutboundGateway gateway = TestUtils.getPropertyValue(gateway1, "handler", SftpOutboundGateway.class); assertEquals("X", TestUtils.getPropertyValue(gateway, "remoteFileSeparator")); - assertNotNull(TestUtils.getPropertyValue(gateway, "sessionFactory")); + assertNotNull(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory")); assertNotNull(TestUtils.getPropertyValue(gateway, "outputChannel")); assertEquals("local-test-dir", TestUtils.getPropertyValue(gateway, "localDirectoryExpression.literalValue")); assertFalse((Boolean) TestUtils.getPropertyValue(gateway, "autoCreateLocalDirectory")); @@ -94,8 +93,8 @@ public class SftpOutboundGatewayParserTests { SftpOutboundGateway gateway = TestUtils.getPropertyValue(gateway2, "handler", SftpOutboundGateway.class); assertEquals("X", TestUtils.getPropertyValue(gateway, "remoteFileSeparator")); - assertNotNull(TestUtils.getPropertyValue(gateway, "sessionFactory")); - assertTrue(TestUtils.getPropertyValue(gateway, "sessionFactory") instanceof CachingSessionFactory); + assertNotNull(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory")); + assertTrue(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory") instanceof CachingSessionFactory); assertNotNull(TestUtils.getPropertyValue(gateway, "outputChannel")); assertEquals("local-test-dir", TestUtils.getPropertyValue(gateway, "localDirectoryExpression.literalValue")); assertFalse((Boolean) TestUtils.getPropertyValue(gateway, "autoCreateLocalDirectory")); @@ -125,7 +124,7 @@ public class SftpOutboundGatewayParserTests { public void testGatewayMv() { SftpOutboundGateway gateway = TestUtils.getPropertyValue(gateway3, "handler", SftpOutboundGateway.class); - assertNotNull(TestUtils.getPropertyValue(gateway, "sessionFactory")); + assertNotNull(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory")); assertNotNull(TestUtils.getPropertyValue(gateway, "outputChannel")); assertEquals(Command.MV, TestUtils.getPropertyValue(gateway, "command")); assertEquals("'foo'", TestUtils.getPropertyValue(gateway, "renameProcessor.expression.expression")); From 836c8e25568caa4e5ab52688b3cdf191eaacf13a Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Fri, 15 Nov 2013 15:21:44 +0200 Subject: [PATCH 04/29] INT-2269 Add ReplyChannelRegistry Allows reply channel resolution after a message has been serialized somewhere in a flow. Previously, the reply channel was lost. With this change, the reply channel can be registered and the header becomes a string (channel name) that can be serialized. JIRA: https://jira.springsource.org/browse/INT-2269 INT-2269 Rename Registry to HeaderChannelRegistry - Extract interface - DefaultHeaderChannelRgistry INT-2269 Polishing; PR Comments - Add errorChannel registration as well. INT-2269 HeaderChannelRegistry - Fix The internal BridgeHandler in MessagingGatewaySupport did not have a channel resolver. When a reply was explicitly routed to the gateway's reply channel, the String representation of the reply channel could not be resolved to a channel. Set the BeanFactory on the bridge handler. Add tests. INT-2269 HeaderChannelRegistry - Fix JMS/Enricher The JMS inbound gateway and ContentEnricher instantiate a MessagingGatewaySupport internally it does not get a reference to the BeanFactory. This means that the internal BridgeHandler cannot resolve the String representation of the reply channel to a channel. Make ChannelPublishingJmsMessageListener BeanFactory aware, and propagate the bean factory to the MGS. Set the MGS bean factory in the ContentEnricher (which is already BFA). INT-2269: Polishing --- .../DefaultHeaderChannelRegistry.java | 249 +++++++++++++++ .../registry/HeaderChannelRegistry.java | 63 ++++ .../AbstractIntegrationNamespaceHandler.java | 39 ++- ...actPollingInboundChannelAdapterParser.java | 1 - .../xml/HeaderEnricherParserSupport.java | 284 ++++++++++-------- .../context/IntegrationContextUtils.java | 3 + .../gateway/MessagingGatewaySupport.java | 22 +- .../channel/BeanFactoryChannelResolver.java | 38 ++- .../transformer/ContentEnricher.java | 7 + .../config/xml/spring-integration-3.0.xsd | 9 + .../HeaderChannelRegistryTests-context.xml | 69 +++++ .../registry/HeaderChannelRegistryTests.java | 167 ++++++++++ .../config/xml/ControlBusTests.java | 31 +- .../xml/EnricherParserTests-context.xml | 10 +- .../config/xml/EnricherParserTests.java | 9 +- .../ChannelPublishingJmsMessageListener.java | 71 +++-- ...waySerializedReplyChannelTests-context.xml | 37 +++ .../GatewaySerializedReplyChannelTests.java | 55 ++++ src/reference/docbook/content-enrichment.xml | 51 +++- src/reference/docbook/message-store.xml | 20 +- src/reference/docbook/whats-new.xml | 10 + 21 files changed, 1060 insertions(+), 185 deletions(-) create mode 100644 spring-integration-core/src/main/java/org/springframework/integration/channel/registry/DefaultHeaderChannelRegistry.java create mode 100644 spring-integration-core/src/main/java/org/springframework/integration/channel/registry/HeaderChannelRegistry.java create mode 100644 spring-integration-core/src/test/java/org/springframework/integration/channel/registry/HeaderChannelRegistryTests-context.xml create mode 100644 spring-integration-core/src/test/java/org/springframework/integration/channel/registry/HeaderChannelRegistryTests.java create mode 100644 spring-integration-jms/src/test/java/org/springframework/integration/jms/request_reply/GatewaySerializedReplyChannelTests-context.xml create mode 100644 spring-integration-jms/src/test/java/org/springframework/integration/jms/request_reply/GatewaySerializedReplyChannelTests.java diff --git a/spring-integration-core/src/main/java/org/springframework/integration/channel/registry/DefaultHeaderChannelRegistry.java b/spring-integration-core/src/main/java/org/springframework/integration/channel/registry/DefaultHeaderChannelRegistry.java new file mode 100644 index 0000000000..3aa7050153 --- /dev/null +++ b/spring-integration-core/src/main/java/org/springframework/integration/channel/registry/DefaultHeaderChannelRegistry.java @@ -0,0 +1,249 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.channel.registry; + +import java.util.Date; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicLong; + +import org.springframework.context.SmartLifecycle; +import org.springframework.integration.MessageChannel; +import org.springframework.integration.context.IntegrationObjectSupport; +import org.springframework.integration.support.channel.BeanFactoryChannelResolver; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.util.Assert; + +/** + * Converts a channel to a name, retaining a reference to the channel keyed by the name. + * Allows a downstream {@link BeanFactoryChannelResolver} to find the channel by name + * in the event that the flow serialized the message at some point. + * Channels are expired after a configurable delay (60 seconds by default). + * The actual average expiry time will be 1.5x the delay. + * + * @author Gary Russell + * @since 3.0 + * + */ +public class DefaultHeaderChannelRegistry extends IntegrationObjectSupport + implements HeaderChannelRegistry, SmartLifecycle, Runnable { + + private static final int DEFAULT_REAPER_DELAY = 60000; + + private final Map channels = new ConcurrentHashMap(); + + private static final AtomicLong id = new AtomicLong(); + + private final String uuid = UUID.randomUUID().toString() + ":"; + + private volatile long reaperDelay; + + private volatile ScheduledFuture reaperScheduledFuture; + + private volatile boolean running; + + private volatile int phase; + + private volatile boolean autoStartup = true; + + /** + * Constructs a registry with the default delay for channel expiry. + */ + public DefaultHeaderChannelRegistry() { + this(DEFAULT_REAPER_DELAY); + } + + /** + * Constructs a registry with the provided delay (milliseconds) for + * channel expiry. + * + * @param reaperDelay the delay in milliseconds. + */ + public DefaultHeaderChannelRegistry(long reaperDelay) { + this.setReaperDelay(reaperDelay); + } + + /** + * Set the reaper delay. + * + * @param reaperDelay the delay in milliseconds. + */ + public final void setReaperDelay(long reaperDelay) { + Assert.isTrue(reaperDelay > 0, "'reaperDelay' must be > 0"); + this.reaperDelay = reaperDelay; + } + + public final long getReaperDelay() { + return reaperDelay; + } + + @Override + public void setTaskScheduler(TaskScheduler taskScheduler) { + super.setTaskScheduler(taskScheduler); + } + + @Override + public int getPhase() { + return this.phase; + } + + public final void setPhase(int phase) { + this.phase = phase; + } + + @Override + public boolean isAutoStartup() { + return this.autoStartup; + } + + public final void setAutoStartup(boolean autoStartup) { + this.autoStartup = autoStartup; + } + + @Override + public final int size() { + return this.channels.size(); + } + + @Override + protected void onInit() throws Exception { + super.onInit(); + Assert.notNull(this.getTaskScheduler(), "a task scheduler is required"); + } + + @Override + public synchronized void start() { + if (!this.running) { + Assert.notNull(this.getTaskScheduler(), "a task scheduler is required"); + this.reaperScheduledFuture = this.getTaskScheduler().schedule(this, + new Date(System.currentTimeMillis() + this.reaperDelay)); + this.running = true; + } + } + + @Override + public synchronized void stop() { + this.running = false; + if (this.reaperScheduledFuture != null) { + this.reaperScheduledFuture.cancel(true); + } + } + + @Override + public void stop(Runnable callback) { + this.stop(); + callback.run(); + } + + @Override + public boolean isRunning() { + return this.running; + } + + @Override + public Object channelToChannelName(Object channel) { + if (channel != null && channel instanceof MessageChannel) { + String name = this.uuid + DefaultHeaderChannelRegistry.id.incrementAndGet(); + channels.put(name, new MessageChannelWrapper((MessageChannel) channel)); + if (logger.isDebugEnabled()) { + logger.debug("Registered " + channel + " as " + name); + } + return name; + } + else { + return channel; + } + } + + @Override + public MessageChannel channelNameToChannel(String name) { + if (name != null) { + MessageChannelWrapper messageChannelWrapper = this.channels.get(name); + if (logger.isDebugEnabled() && messageChannelWrapper != null) { + logger.debug("Retrieved " + messageChannelWrapper.getChannel() + " with " + name); + } + return messageChannelWrapper == null ? null : messageChannelWrapper.getChannel(); + } + return null; + } + + /** + * Cancel the scheduled reap task and run immediately; then reschedule. + */ + @Override + public void runReaper() { + synchronized(this) { + this.reaperScheduledFuture.cancel(false); + this.reaperScheduledFuture = null; + } + this.run(); + } + + @Override + public void run() { + this.reaperScheduledFuture = null; + if (logger.isTraceEnabled()) { + logger.trace("Reaper started; channels size=" + this.channels.size()); + } + Iterator> iterator = this.channels.entrySet().iterator(); + long threshold = System.currentTimeMillis() - this.reaperDelay; + while (iterator.hasNext()) { + Entry entry = iterator.next(); + if (entry.getValue().getCreated() < threshold) { + if (logger.isDebugEnabled()) { + logger.debug("Expiring " + entry.getKey() + " (" + entry.getValue().getChannel() + ")"); + } + iterator.remove(); + } + } + synchronized (this) { + if (this.reaperScheduledFuture == null) { + this.reaperScheduledFuture = this.getTaskScheduler().schedule(this, + new Date(System.currentTimeMillis() + this.reaperDelay)); + } + } + if (logger.isTraceEnabled()) { + logger.trace("Reaper completed; channels size=" + this.channels.size()); + } + } + + + private class MessageChannelWrapper { + + private final MessageChannel channel; + + private final long created; + + private MessageChannelWrapper(MessageChannel channel) { + this.channel = channel; + this.created = System.currentTimeMillis(); + } + + public final long getCreated() { + return created; + } + + public final MessageChannel getChannel() { + return channel; + } + + } + +} diff --git a/spring-integration-core/src/main/java/org/springframework/integration/channel/registry/HeaderChannelRegistry.java b/spring-integration-core/src/main/java/org/springframework/integration/channel/registry/HeaderChannelRegistry.java new file mode 100644 index 0000000000..66fd2eb118 --- /dev/null +++ b/spring-integration-core/src/main/java/org/springframework/integration/channel/registry/HeaderChannelRegistry.java @@ -0,0 +1,63 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.channel.registry; + +import org.springframework.integration.MessageChannel; +import org.springframework.integration.support.channel.BeanFactoryChannelResolver; +import org.springframework.jmx.export.annotation.ManagedAttribute; +import org.springframework.jmx.export.annotation.ManagedOperation; + +/** + * Implementations convert a channel to a name, retaining a reference to the channel keyed by the name. + * Allows a downstream {@link BeanFactoryChannelResolver} to find the channel by name in + * the event that the flow serialized the message at some point. + * + * @author Gary Russell + * @since 3.0 + * + */ +public interface HeaderChannelRegistry { + + /** + * Converts the channel to a name (String). If the channel is not a + * {@link MessageChannel}, it is returned unchanged. + * + * @param channel The channel. + * @return The channel name, or the channel if it is not a MessageChannel. + */ + public abstract Object channelToChannelName(Object channel); + + /** + * Converts the channel name back to a {@link MessageChannel} (if it is + * registered). + * @param name The name of the channel. + * @return The channel, or null if there is no channel registered with the name. + */ + public abstract MessageChannel channelNameToChannel(String name); + + /** + * @return the current size of the registry + */ + @ManagedAttribute + public abstract int size(); + + /** + * Cancel the scheduled reap task and run immediately; then reschedule. + */ + @ManagedOperation(description = "Cancel the scheduled reap task and run immediately; then reschedule.") + public abstract void runReaper(); + +} diff --git a/spring-integration-core/src/main/java/org/springframework/integration/config/xml/AbstractIntegrationNamespaceHandler.java b/spring-integration-core/src/main/java/org/springframework/integration/config/xml/AbstractIntegrationNamespaceHandler.java index 104d17e7e4..dd65c4337f 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/config/xml/AbstractIntegrationNamespaceHandler.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/config/xml/AbstractIntegrationNamespaceHandler.java @@ -16,8 +16,6 @@ package org.springframework.integration.config.xml; -import static org.springframework.integration.context.IntegrationContextUtils.INTEGRATION_EVALUATION_CONTEXT_BEAN_NAME; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.w3c.dom.Element; @@ -35,6 +33,7 @@ import org.springframework.beans.factory.xml.BeanDefinitionParser; import org.springframework.beans.factory.xml.NamespaceHandler; import org.springframework.beans.factory.xml.NamespaceHandlerSupport; import org.springframework.beans.factory.xml.ParserContext; +import org.springframework.integration.channel.registry.DefaultHeaderChannelRegistry; import org.springframework.integration.config.IntegrationEvaluationContextFactoryBean; import org.springframework.integration.config.xml.ChannelInitializer.AutoCreateCandidatesCollector; import org.springframework.integration.context.IntegrationContextUtils; @@ -68,15 +67,18 @@ public abstract class AbstractIntegrationNamespaceHandler implements NamespaceHa private final NamespaceHandlerDelegate delegate = new NamespaceHandlerDelegate(); + @Override public final BeanDefinition parse(Element element, ParserContext parserContext) { this.verifySchemaVersion(element, parserContext); this.registerImplicitChannelCreator(parserContext); this.registerIntegrationEvaluationContext(parserContext); + this.registerHeaderChannelRegistry(parserContext); this.registerBuiltInBeans(parserContext); this.registerDefaultConfiguringBeanFactoryPostProcessorIfNecessary(parserContext); return this.delegate.parse(element, parserContext); } + @Override public final BeanDefinitionHolder decorate(Node source, BeanDefinitionHolder definition, ParserContext parserContext) { return this.delegate.decorate(source, definition, parserContext); } @@ -127,10 +129,11 @@ public abstract class AbstractIntegrationNamespaceHandler implements NamespaceHa // unlike DefaultConfiguringBeanFactoryPostProcessor, we need one of these per registry // therefore we need to call containsBeanDefinition(..) which does not consider the parent registry alreadyRegistered = ((ListableBeanFactory) parserContext.getRegistry()).containsBeanDefinition( - INTEGRATION_EVALUATION_CONTEXT_BEAN_NAME); + IntegrationContextUtils.INTEGRATION_EVALUATION_CONTEXT_BEAN_NAME); } else { - alreadyRegistered = parserContext.getRegistry().isBeanNameInUse(INTEGRATION_EVALUATION_CONTEXT_BEAN_NAME); + alreadyRegistered = parserContext.getRegistry().isBeanNameInUse( + IntegrationContextUtils.INTEGRATION_EVALUATION_CONTEXT_BEAN_NAME); } if (!alreadyRegistered) { BeanDefinitionBuilder integrationEvaluationContextBuilder = BeanDefinitionBuilder @@ -195,6 +198,33 @@ public abstract class AbstractIntegrationNamespaceHandler implements NamespaceHa } } + /** + * Register a DefaultHeaderChannelRegistry in the given BeanDefinitionRegistry, if necessary. + */ + private void registerHeaderChannelRegistry(ParserContext parserContext) { + boolean alreadyRegistered = false; + if (parserContext.getRegistry() instanceof ListableBeanFactory) { + alreadyRegistered = ((ListableBeanFactory) parserContext.getRegistry()) + .containsBean(IntegrationContextUtils.INTEGRATION_HEADER_CHANNEL_REGISTRY_BEAN_NAME); + } + else { + alreadyRegistered = parserContext.getRegistry().isBeanNameInUse( + IntegrationContextUtils.INTEGRATION_HEADER_CHANNEL_REGISTRY_BEAN_NAME); + } + if (!alreadyRegistered) { + if (logger.isInfoEnabled()) { + logger.info("No bean named '" + IntegrationContextUtils.INTEGRATION_HEADER_CHANNEL_REGISTRY_BEAN_NAME + + "' has been explicitly defined. Therefore, a default DefaultHeaderChannelRegistry will be created."); + } + BeanDefinitionBuilder schedulerBuilder = BeanDefinitionBuilder.genericBeanDefinition(DefaultHeaderChannelRegistry.class); + BeanDefinitionHolder replyChannelRegistryComponent = new BeanDefinitionHolder( + schedulerBuilder.getBeanDefinition(), + IntegrationContextUtils.INTEGRATION_HEADER_CHANNEL_REGISTRY_BEAN_NAME); + BeanDefinitionReaderUtils.registerBeanDefinition(replyChannelRegistryComponent, parserContext.getRegistry()); + } + } + + protected final void registerBeanDefinitionDecorator(String elementName, BeanDefinitionDecorator decorator) { this.delegate.doRegisterBeanDefinitionDecorator(elementName, decorator); } @@ -225,6 +255,7 @@ public abstract class AbstractIntegrationNamespaceHandler implements NamespaceHa private class NamespaceHandlerDelegate extends NamespaceHandlerSupport { + @Override public void init() { AbstractIntegrationNamespaceHandler.this.init(); } diff --git a/spring-integration-core/src/main/java/org/springframework/integration/config/xml/AbstractPollingInboundChannelAdapterParser.java b/spring-integration-core/src/main/java/org/springframework/integration/config/xml/AbstractPollingInboundChannelAdapterParser.java index 6da6aec529..2e7d981a87 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/config/xml/AbstractPollingInboundChannelAdapterParser.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/config/xml/AbstractPollingInboundChannelAdapterParser.java @@ -37,7 +37,6 @@ import org.springframework.util.xml.DomUtils; public abstract class AbstractPollingInboundChannelAdapterParser extends AbstractChannelAdapterParser { @Override - @SuppressWarnings("unchecked") protected AbstractBeanDefinition doParse(Element element, ParserContext parserContext, String channelName) { BeanMetadataElement source = this.parseSource(element, parserContext); if (source == null) { diff --git a/spring-integration-core/src/main/java/org/springframework/integration/config/xml/HeaderEnricherParserSupport.java b/spring-integration-core/src/main/java/org/springframework/integration/config/xml/HeaderEnricherParserSupport.java index 5dba32f685..fba8d51b2e 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/config/xml/HeaderEnricherParserSupport.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/config/xml/HeaderEnricherParserSupport.java @@ -29,11 +29,12 @@ import org.springframework.beans.factory.config.TypedStringValue; import org.springframework.beans.factory.support.BeanDefinitionBuilder; import org.springframework.beans.factory.support.ManagedMap; import org.springframework.beans.factory.xml.ParserContext; +import org.springframework.integration.context.IntegrationContextUtils; +import org.springframework.integration.expression.DynamicExpression; +import org.springframework.integration.transformer.HeaderEnricher; import org.springframework.util.ClassUtils; import org.springframework.util.StringUtils; import org.springframework.util.xml.DomUtils; -import org.springframework.integration.expression.DynamicExpression; -import org.springframework.integration.transformer.HeaderEnricher; /** * Base support class for 'header-enricher' parsers. @@ -41,6 +42,7 @@ import org.springframework.integration.transformer.HeaderEnricher; * @author Mark Fisher * @author Oleg Zhurakousky * @author Artem Bilan + * @author Gary Russell * @since 2.0 */ public abstract class HeaderEnricherParserSupport extends AbstractTransformerParser { @@ -49,6 +51,16 @@ public abstract class HeaderEnricherParserSupport extends AbstractTransformerPar private final Map> elementToTypeMap = new HashMap>(); + private final static Map cannedHeaderElementExpressions = new HashMap(); + + static { + cannedHeaderElementExpressions.put("header-channels-to-string", new String[][] { + {"replyChannel", "@" + IntegrationContextUtils.INTEGRATION_HEADER_CHANNEL_REGISTRY_BEAN_NAME + + ".channelToChannelName(headers.replyChannel)" }, + {"errorChannel", "@" + IntegrationContextUtils.INTEGRATION_HEADER_CHANNEL_REGISTRY_BEAN_NAME + + ".channelToChannelName(headers.errorChannel)" }, + }); + } @Override protected final String getTransformerClassName() { @@ -86,6 +98,8 @@ public abstract class HeaderEnricherParserSupport extends AbstractTransformerPar Element headerElement = (Element) node; String elementName = node.getLocalName(); Class headerType = null; + String expression = null; + String overwrite = headerElement.getAttribute("overwrite"); if ("header".equals(elementName)) { headerName = headerElement.getAttribute(NAME_ATTRIBUTE); } @@ -114,135 +128,157 @@ public abstract class HeaderEnricherParserSupport extends AbstractTransformerPar } } } - if (headerName != null) { - String value = headerElement.getAttribute("value"); - String ref = headerElement.getAttribute(REF_ATTRIBUTE); - String method = headerElement.getAttribute(METHOD_ATTRIBUTE); - String expression = headerElement.getAttribute(EXPRESSION_ATTRIBUTE); - - Element beanElement = null; - Element scriptElement = null; - Element expressionElement = null; - - List subElements = DomUtils.getChildElements(headerElement); - if (!subElements.isEmpty()) { - Element subElement = subElements.get(0); - String subElementLocalName = subElement.getLocalName(); - if ("bean".equals(subElementLocalName)) { - beanElement = subElement; - } - else if ("script".equals(subElementLocalName)) { - scriptElement = subElement; - } - else if ("expression".equals(subElementLocalName)) { - expressionElement = subElement; - } - if (beanElement == null && scriptElement == null && expressionElement == null) { - parserContext.getReaderContext().error("Only 'bean', 'script' or 'expression' can be defined as a sub-element", element); + if (headerName == null) { + if (cannedHeaderElementExpressions.containsKey(elementName)) { + for (int j = 0; j < cannedHeaderElementExpressions.get(elementName).length; j++) { + headerName = cannedHeaderElementExpressions.get(elementName)[j][0]; + expression = cannedHeaderElementExpressions.get(elementName)[j][1]; + overwrite = "true"; + this.addHeader(element, headers, parserContext, headerName, headerElement, headerType, + expression, overwrite); } } - if (StringUtils.hasText(expression) && expressionElement != null) { - parserContext.getReaderContext().error("The 'expression' attribute and sub-element are mutually exclusive", element); - } - - boolean isValue = StringUtils.hasText(value); - boolean isRef = StringUtils.hasText(ref); - boolean hasMethod = StringUtils.hasText(method); - boolean isExpression = StringUtils.hasText(expression) || expressionElement != null; - boolean isScript = scriptElement != null; - - BeanDefinition innerComponentDefinition = null; - - if (beanElement != null) { - innerComponentDefinition = parserContext.getDelegate().parseBeanDefinitionElement(beanElement).getBeanDefinition(); - } - else if (isScript) { - innerComponentDefinition = parserContext.getDelegate().parseCustomElement(scriptElement); - } - - boolean isCustomBean = innerComponentDefinition != null; - - if (hasMethod && isScript) { - parserContext.getReaderContext().error("The 'method' attribute cannot be used when a 'script' sub-element is defined", element); - } - - if (!(isValue ^ (isRef ^ (isExpression ^ isCustomBean)))) { - parserContext.getReaderContext().error( - "Exactly one of the 'ref', 'value', 'expression' or inner bean is required.", element); - } - BeanDefinitionBuilder valueProcessorBuilder = null; - if (isValue) { - if (hasMethod) { - parserContext.getReaderContext().error( - "The 'method' attribute cannot be used with the 'value' attribute.", element); - } - Object headerValue = (headerType != null) ? - new TypedStringValue(value, headerType) : value; - valueProcessorBuilder = BeanDefinitionBuilder.genericBeanDefinition( - IntegrationNamespaceUtils.BASE_PACKAGE + ".transformer.support.StaticHeaderValueMessageProcessor"); - valueProcessorBuilder.addConstructorArgValue(headerValue); - } - else if (isExpression) { - if (hasMethod) { - parserContext.getReaderContext().error( - "The 'method' attribute cannot be used with the 'expression' attribute.", element); - } - valueProcessorBuilder = BeanDefinitionBuilder.genericBeanDefinition( - IntegrationNamespaceUtils.BASE_PACKAGE + ".transformer.support.ExpressionEvaluatingHeaderValueMessageProcessor"); - if (expressionElement != null) { - BeanDefinitionBuilder dynamicExpressionBuilder = BeanDefinitionBuilder.genericBeanDefinition(DynamicExpression.class); - dynamicExpressionBuilder.addConstructorArgValue(expressionElement.getAttribute("key")); - dynamicExpressionBuilder.addConstructorArgReference(expressionElement.getAttribute("source")); - valueProcessorBuilder.addConstructorArgValue(dynamicExpressionBuilder.getBeanDefinition()); - } - else { - valueProcessorBuilder.addConstructorArgValue(expression); - } - valueProcessorBuilder.addConstructorArgValue(headerType); - } - else if (isCustomBean) { - if (StringUtils.hasText(headerElement.getAttribute("type"))) { - parserContext.getReaderContext().error( - "The 'type' attribute cannot be used with an inner bean.", element); - } - if (hasMethod || isScript) { - valueProcessorBuilder = BeanDefinitionBuilder.genericBeanDefinition( - IntegrationNamespaceUtils.BASE_PACKAGE + ".transformer.support.MessageProcessingHeaderValueMessageProcessor"); - valueProcessorBuilder.addConstructorArgValue(innerComponentDefinition); - if (hasMethod) { - valueProcessorBuilder.addConstructorArgValue(method); - } - } - else { - valueProcessorBuilder = BeanDefinitionBuilder.genericBeanDefinition( - IntegrationNamespaceUtils.BASE_PACKAGE + ".transformer.support.StaticHeaderValueMessageProcessor"); - valueProcessorBuilder.addConstructorArgValue(innerComponentDefinition); - } - } - else { - if (StringUtils.hasText(headerElement.getAttribute("type"))) { - parserContext.getReaderContext().error( - "The 'type' attribute cannot be used with the 'ref' attribute.", element); - } - if (hasMethod) { - valueProcessorBuilder = BeanDefinitionBuilder.genericBeanDefinition( - IntegrationNamespaceUtils.BASE_PACKAGE + ".transformer.support.MessageProcessingHeaderValueMessageProcessor"); - valueProcessorBuilder.addConstructorArgReference(ref); - valueProcessorBuilder.addConstructorArgValue(method); - } - else { - valueProcessorBuilder = BeanDefinitionBuilder.genericBeanDefinition( - IntegrationNamespaceUtils.BASE_PACKAGE + ".transformer.support.StaticHeaderValueMessageProcessor"); - valueProcessorBuilder.addConstructorArgReference(ref); - } - } - IntegrationNamespaceUtils.setValueIfAttributeDefined(valueProcessorBuilder, headerElement, "overwrite"); - headers.put(headerName, valueProcessorBuilder.getBeanDefinition()); + } + else { + this.addHeader(element, headers, parserContext, headerName, headerElement, headerType, expression, + overwrite); } } } } + private void addHeader(Element element, ManagedMap headers, ParserContext parserContext, + String headerName, Element headerElement, Class headerType, String expression, String overwrite) { + + String value = headerElement.getAttribute("value"); + String ref = headerElement.getAttribute(REF_ATTRIBUTE); + String method = headerElement.getAttribute(METHOD_ATTRIBUTE); + if (expression == null) { + expression = headerElement.getAttribute(EXPRESSION_ATTRIBUTE); + } + + Element beanElement = null; + Element scriptElement = null; + Element expressionElement = null; + + List subElements = DomUtils.getChildElements(headerElement); + if (!subElements.isEmpty()) { + Element subElement = subElements.get(0); + String subElementLocalName = subElement.getLocalName(); + if ("bean".equals(subElementLocalName)) { + beanElement = subElement; + } + else if ("script".equals(subElementLocalName)) { + scriptElement = subElement; + } + else if ("expression".equals(subElementLocalName)) { + expressionElement = subElement; + } + if (beanElement == null && scriptElement == null && expressionElement == null) { + parserContext.getReaderContext().error("Only 'bean', 'script' or 'expression' can be defined as a sub-element", element); + } + } + if (StringUtils.hasText(expression) && expressionElement != null) { + parserContext.getReaderContext().error("The 'expression' attribute and sub-element are mutually exclusive", element); + } + + boolean isValue = StringUtils.hasText(value); + boolean isRef = StringUtils.hasText(ref); + boolean hasMethod = StringUtils.hasText(method); + boolean isExpression = StringUtils.hasText(expression) || expressionElement != null; + boolean isScript = scriptElement != null; + + BeanDefinition innerComponentDefinition = null; + + if (beanElement != null) { + innerComponentDefinition = parserContext.getDelegate().parseBeanDefinitionElement(beanElement).getBeanDefinition(); + } + else if (isScript) { + innerComponentDefinition = parserContext.getDelegate().parseCustomElement(scriptElement); + } + + boolean isCustomBean = innerComponentDefinition != null; + + if (hasMethod && isScript) { + parserContext.getReaderContext().error("The 'method' attribute cannot be used when a 'script' sub-element is defined", element); + } + + if (!(isValue ^ (isRef ^ (isExpression ^ isCustomBean)))) { + parserContext.getReaderContext().error( + "Exactly one of the 'ref', 'value', 'expression' or inner bean is required.", element); + } + BeanDefinitionBuilder valueProcessorBuilder = null; + if (isValue) { + if (hasMethod) { + parserContext.getReaderContext().error( + "The 'method' attribute cannot be used with the 'value' attribute.", element); + } + Object headerValue = (headerType != null) ? + new TypedStringValue(value, headerType) : value; + valueProcessorBuilder = BeanDefinitionBuilder.genericBeanDefinition( + IntegrationNamespaceUtils.BASE_PACKAGE + ".transformer.support.StaticHeaderValueMessageProcessor"); + valueProcessorBuilder.addConstructorArgValue(headerValue); + } + else if (isExpression) { + if (hasMethod) { + parserContext.getReaderContext().error( + "The 'method' attribute cannot be used with the 'expression' attribute.", element); + } + valueProcessorBuilder = BeanDefinitionBuilder.genericBeanDefinition( + IntegrationNamespaceUtils.BASE_PACKAGE + ".transformer.support.ExpressionEvaluatingHeaderValueMessageProcessor"); + if (expressionElement != null) { + BeanDefinitionBuilder dynamicExpressionBuilder = BeanDefinitionBuilder.genericBeanDefinition(DynamicExpression.class); + dynamicExpressionBuilder.addConstructorArgValue(expressionElement.getAttribute("key")); + dynamicExpressionBuilder.addConstructorArgReference(expressionElement.getAttribute("source")); + valueProcessorBuilder.addConstructorArgValue(dynamicExpressionBuilder.getBeanDefinition()); + } + else { + valueProcessorBuilder.addConstructorArgValue(expression); + } + valueProcessorBuilder.addConstructorArgValue(headerType); + } + else if (isCustomBean) { + if (StringUtils.hasText(headerElement.getAttribute("type"))) { + parserContext.getReaderContext().error( + "The 'type' attribute cannot be used with an inner bean.", element); + } + if (hasMethod || isScript) { + valueProcessorBuilder = BeanDefinitionBuilder.genericBeanDefinition( + IntegrationNamespaceUtils.BASE_PACKAGE + ".transformer.support.MessageProcessingHeaderValueMessageProcessor"); + valueProcessorBuilder.addConstructorArgValue(innerComponentDefinition); + if (hasMethod) { + valueProcessorBuilder.addConstructorArgValue(method); + } + } + else { + valueProcessorBuilder = BeanDefinitionBuilder.genericBeanDefinition( + IntegrationNamespaceUtils.BASE_PACKAGE + ".transformer.support.StaticHeaderValueMessageProcessor"); + valueProcessorBuilder.addConstructorArgValue(innerComponentDefinition); + } + } + else { + if (StringUtils.hasText(headerElement.getAttribute("type"))) { + parserContext.getReaderContext().error( + "The 'type' attribute cannot be used with the 'ref' attribute.", element); + } + if (hasMethod) { + valueProcessorBuilder = BeanDefinitionBuilder.genericBeanDefinition( + IntegrationNamespaceUtils.BASE_PACKAGE + ".transformer.support.MessageProcessingHeaderValueMessageProcessor"); + valueProcessorBuilder.addConstructorArgReference(ref); + valueProcessorBuilder.addConstructorArgValue(method); + } + else { + valueProcessorBuilder = BeanDefinitionBuilder.genericBeanDefinition( + IntegrationNamespaceUtils.BASE_PACKAGE + ".transformer.support.StaticHeaderValueMessageProcessor"); + valueProcessorBuilder.addConstructorArgReference(ref); + } + } + if (StringUtils.hasText(overwrite)) { + valueProcessorBuilder.addPropertyValue("overwrite", overwrite); + } + headers.put(headerName, valueProcessorBuilder.getBeanDefinition()); + } + /** * Subclasses may override this method to provide any additional processing. */ diff --git a/spring-integration-core/src/main/java/org/springframework/integration/context/IntegrationContextUtils.java b/spring-integration-core/src/main/java/org/springframework/integration/context/IntegrationContextUtils.java index a3d6f70a0a..f432a0ea5e 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/context/IntegrationContextUtils.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/context/IntegrationContextUtils.java @@ -45,6 +45,9 @@ public abstract class IntegrationContextUtils { public static final String INTEGRATION_EVALUATION_CONTEXT_BEAN_NAME = "integrationEvaluationContext"; + public static final String INTEGRATION_HEADER_CHANNEL_REGISTRY_BEAN_NAME = "integrationHeaderChannelRegistry"; + + /** * Return the {@link MetadataStore} bean whose name is "metadataStore". * @param beanFactory BeanFactory for lookup, must not be null. diff --git a/spring-integration-core/src/main/java/org/springframework/integration/gateway/MessagingGatewaySupport.java b/spring-integration-core/src/main/java/org/springframework/integration/gateway/MessagingGatewaySupport.java index 809770fa1d..a3d840822a 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/gateway/MessagingGatewaySupport.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/gateway/MessagingGatewaySupport.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2010 the original author or authors. + * Copyright 2002-2013 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ package org.springframework.integration.gateway; import org.springframework.integration.Message; import org.springframework.integration.MessageChannel; import org.springframework.integration.MessagingException; -import org.springframework.integration.core.MessageHandler; import org.springframework.integration.core.MessagingTemplate; import org.springframework.integration.core.PollableChannel; import org.springframework.integration.core.SubscribableChannel; @@ -41,8 +40,9 @@ import org.springframework.util.Assert; * {@link MessageChannel}s for sending, receiving, or request-reply operations. * Exposes setters for configuring request and reply {@link MessageChannel}s as * well as the timeout values for sending and receiving Messages. - * + * * @author Mark Fisher + * @author Gary Russell */ public abstract class MessagingGatewaySupport extends AbstractEndpoint implements TrackableComponent { @@ -84,7 +84,7 @@ public abstract class MessagingGatewaySupport extends AbstractEndpoint implement /** * Set the request channel. - * + * * @param requestChannel the channel to which request messages will be sent */ public void setRequestChannel(MessageChannel requestChannel) { @@ -94,7 +94,7 @@ public abstract class MessagingGatewaySupport extends AbstractEndpoint implement /** * Set the reply channel. If no reply channel is provided, this gateway will * always use an anonymous, temporary channel for handling replies. - * + * * @param replyChannel the channel from which reply messages will be received */ public void setReplyChannel(MessageChannel replyChannel) { @@ -113,7 +113,7 @@ public abstract class MessagingGatewaySupport extends AbstractEndpoint implement /** * Set the timeout value for sending request messages. If not * explicitly configured, the default is one second. - * + * * @param requestTimeout the timeout value in milliseconds */ public void setRequestTimeout(long requestTimeout) { @@ -123,7 +123,7 @@ public abstract class MessagingGatewaySupport extends AbstractEndpoint implement /** * Set the timeout value for receiving reply messages. If not * explicitly configured, the default is one second. - * + * * @param replyTimeout the timeout value in milliseconds */ public void setReplyTimeout(long replyTimeout) { @@ -153,6 +153,7 @@ public abstract class MessagingGatewaySupport extends AbstractEndpoint implement * Specify whether this gateway should be tracked in the Message History * of Messages that originate from its send or sendAndReceive operations. */ + @Override public void setShouldTrack(boolean shouldTrack) { this.historyWritingPostProcessor.setShouldTrack(shouldTrack); } @@ -283,7 +284,11 @@ public abstract class MessagingGatewaySupport extends AbstractEndpoint implement return; } AbstractEndpoint correlator = null; - MessageHandler handler = new BridgeHandler(); + BridgeHandler handler = new BridgeHandler(); + if (this.getBeanFactory() != null) { + handler.setBeanFactory(this.getBeanFactory()); + } + handler.afterPropertiesSet(); if (this.replyChannel instanceof SubscribableChannel) { correlator = new EventDrivenConsumer( (SubscribableChannel) this.replyChannel, handler); @@ -320,6 +325,7 @@ public abstract class MessagingGatewaySupport extends AbstractEndpoint implement private static class DefaultRequestMapper implements InboundMessageMapper { + @Override public Message toMessage(Object object) throws Exception { if (object instanceof Message) { return (Message) object; diff --git a/spring-integration-core/src/main/java/org/springframework/integration/support/channel/BeanFactoryChannelResolver.java b/spring-integration-core/src/main/java/org/springframework/integration/support/channel/BeanFactoryChannelResolver.java index 8f9abe8e95..2a17a0e139 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/support/channel/BeanFactoryChannelResolver.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/support/channel/BeanFactoryChannelResolver.java @@ -16,25 +16,33 @@ package org.springframework.integration.support.channel; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + import org.springframework.beans.BeansException; import org.springframework.beans.factory.BeanFactory; import org.springframework.beans.factory.BeanFactoryAware; import org.springframework.integration.MessageChannel; +import org.springframework.integration.channel.registry.HeaderChannelRegistry; +import org.springframework.integration.context.IntegrationContextUtils; import org.springframework.util.Assert; /** * {@link ChannelResolver} implementation based on a Spring {@link BeanFactory}. - * + * *

Will lookup Spring managed beans identified by bean name, * expecting them to be of type {@link MessageChannel}. - * + * * @author Mark Fisher * @see org.springframework.beans.factory.BeanFactory */ public class BeanFactoryChannelResolver implements ChannelResolver, BeanFactoryAware { + private final static Log logger = LogFactory.getLog(BeanFactoryChannelResolver.class); + private volatile BeanFactory beanFactory; + private volatile HeaderChannelRegistry replyChannelRegistry; /** * Create a new instance of the {@link BeanFactoryChannelResolver} class. @@ -53,25 +61,45 @@ public class BeanFactoryChannelResolver implements ChannelResolver, BeanFactoryA * replaced by the {@link BeanFactory} that creates it (c.f. the * {@link BeanFactoryAware} contract). So only use this constructor if you * are instantiating this object explicitly rather than defining a bean. - * + * * @param beanFactory the bean factory to be used to lookup {@link MessageChannel}s. */ public BeanFactoryChannelResolver(BeanFactory beanFactory) { Assert.notNull(beanFactory, "BeanFactory must not be null"); - this.beanFactory = beanFactory; + this.lookupHeaderChannelRegistry(beanFactory); } - + @Override public void setBeanFactory(BeanFactory beanFactory) { this.beanFactory = beanFactory; + this.lookupHeaderChannelRegistry(beanFactory); } + private void lookupHeaderChannelRegistry(BeanFactory beanFactory) { + this.beanFactory = beanFactory; + try { + this.replyChannelRegistry = beanFactory.getBean( + IntegrationContextUtils.INTEGRATION_HEADER_CHANNEL_REGISTRY_BEAN_NAME, + HeaderChannelRegistry.class); + } + catch (Exception e) { + logger.warn("No HeaderChannelRegistry found", e); + } + } + + @Override public MessageChannel resolveChannelName(String name) { Assert.state(this.beanFactory != null, "BeanFactory is required"); try { return this.beanFactory.getBean(name, MessageChannel.class); } catch (BeansException e) { + if (this.replyChannelRegistry != null) { + MessageChannel channel = this.replyChannelRegistry.channelNameToChannel(name); + if (channel != null) { + return channel; + } + } throw new ChannelResolutionException( "failed to look up MessageChannel bean with name '" + name + "'", e); } diff --git a/spring-integration-core/src/main/java/org/springframework/integration/transformer/ContentEnricher.java b/spring-integration-core/src/main/java/org/springframework/integration/transformer/ContentEnricher.java index b6d18c5173..21c60f32e4 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/transformer/ContentEnricher.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/transformer/ContentEnricher.java @@ -214,6 +214,10 @@ public class ContentEnricher extends AbstractReplyProducingMessageHandler implem this.gateway.setReplyChannel(replyChannel); } + if (this.getBeanFactory() != null) { + this.gateway.setBeanFactory(this.getBeanFactory()); + } + this.gateway.afterPropertiesSet(); } @@ -294,6 +298,7 @@ public class ContentEnricher extends AbstractReplyProducingMessageHandler implem * Lifecycle implementation. If no requestChannel is defined, this method * has no effect as in that case no Gateway is initialized. */ + @Override public void start() { if (this.gateway != null) { this.gateway.start(); @@ -304,6 +309,7 @@ public class ContentEnricher extends AbstractReplyProducingMessageHandler implem * Lifecycle implementation. If no requestChannel is defined, this method * has no effect as in that case no Gateway is initialized. */ + @Override public void stop() { if (this.gateway != null) { this.gateway.stop(); @@ -314,6 +320,7 @@ public class ContentEnricher extends AbstractReplyProducingMessageHandler implem * Lifecycle implementation. If no requestChannel is defined, this method * will return always return true as no Gateway is initialized. */ + @Override public boolean isRunning() { if (this.gateway != null) { return this.gateway.isRunning(); diff --git a/spring-integration-core/src/main/resources/org/springframework/integration/config/xml/spring-integration-3.0.xsd b/spring-integration-core/src/main/resources/org/springframework/integration/config/xml/spring-integration-3.0.xsd index a7b47c6edb..d715557a4f 100644 --- a/spring-integration-core/src/main/resources/org/springframework/integration/config/xml/spring-integration-3.0.xsd +++ b/spring-integration-core/src/main/resources/org/springframework/integration/config/xml/spring-integration-3.0.xsd @@ -1853,6 +1853,15 @@ + + + + Converts the 'replyChannel' and 'errorChannel' headers to a String after registering it in the HeaderChannelRegistry. + Use this when a message is serialized for any reason. No changes are made + if the header does not exist, or if the header does not currently reference a MessageChannel. + + + diff --git a/spring-integration-core/src/test/java/org/springframework/integration/channel/registry/HeaderChannelRegistryTests-context.xml b/spring-integration-core/src/test/java/org/springframework/integration/channel/registry/HeaderChannelRegistryTests-context.xml new file mode 100644 index 0000000000..5e073d435e --- /dev/null +++ b/spring-integration-core/src/test/java/org/springframework/integration/channel/registry/HeaderChannelRegistryTests-context.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spring-integration-core/src/test/java/org/springframework/integration/channel/registry/HeaderChannelRegistryTests.java b/spring-integration-core/src/test/java/org/springframework/integration/channel/registry/HeaderChannelRegistryTests.java new file mode 100644 index 0000000000..363f154299 --- /dev/null +++ b/spring-integration-core/src/test/java/org/springframework/integration/channel/registry/HeaderChannelRegistryTests.java @@ -0,0 +1,167 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.channel.registry; + +import static org.hamcrest.Matchers.instanceOf; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.hamcrest.Matchers; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.integration.Message; +import org.springframework.integration.MessageChannel; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.channel.MessagePublishingErrorHandler; +import org.springframework.integration.channel.QueueChannel; +import org.springframework.integration.core.MessagingTemplate; +import org.springframework.integration.handler.AbstractReplyProducingMessageHandler; +import org.springframework.integration.message.ErrorMessage; +import org.springframework.integration.message.GenericMessage; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +/** + * @author Gary Russell + * @since 3.0 + * + */ +@ContextConfiguration +@RunWith(SpringJUnit4ClassRunner.class) +public class HeaderChannelRegistryTests { + + @Autowired + MessageChannel input; + + @Autowired + MessageChannel inputPolled; + + @Autowired + QueueChannel alreadyAString; + + @Autowired + TaskScheduler taskScheduler; + + @Autowired + Gateway gatewayNoReplyChannel; + + @Autowired + Gateway gatewayExplicitReplyChannel; + + @Test + public void testReplace() { + MessagingTemplate template = new MessagingTemplate(); + template.setDefaultChannel(this.input); + Message reply = template.sendAndReceive(new GenericMessage("foo")); + assertNotNull(reply); + assertEquals("echo:foo", reply.getPayload()); + } + + @Test + public void testReplaceGatewayWithNoReplyChannel() { + String reply = this.gatewayNoReplyChannel.exchange("foo"); + assertNotNull(reply); + assertEquals("echo:foo", reply); + } + + @Test + public void testReplaceGatewayWithExplicitReplyChannel() { + String reply = this.gatewayExplicitReplyChannel.exchange("foo"); + assertNotNull(reply); + assertEquals("echo:foo", reply); + } + + /** + * MessagingTemplate sets the errorChannel to the replyChannel so it gets any async + * exceptions via the default {@link MessagePublishingErrorHandler}. + */ + @Test + public void testReplaceError() { + MessagingTemplate template = new MessagingTemplate(); + template.setDefaultChannel(this.inputPolled); + Message reply = template.sendAndReceive(new GenericMessage("bar")); + assertNotNull(reply); + assertTrue(reply instanceof ErrorMessage); + } + + @Test + public void testAlreadyAString() { + Message requestMessage = MessageBuilder.withPayload("foo") + .setReplyChannelName("alreadyAString") + .setErrorChannelName("alreadyAnotherString") + .build(); + this.input.send(requestMessage); + Message reply = alreadyAString.receive(0); + assertNotNull(reply); + assertEquals("echo:foo", reply.getPayload()); + } + + @Test + public void testNull() { + Message requestMessage = MessageBuilder.withPayload("foo") + .build(); + try { + this.input.send(requestMessage); + fail("expected exception"); + } + catch (Exception e) { + assertThat(e.getMessage(), Matchers.containsString("no output-channel or replyChannel")); + } + } + + @Test + public void testExpire() throws Exception { + DefaultHeaderChannelRegistry registry = new DefaultHeaderChannelRegistry(50); + registry.setTaskScheduler(this.taskScheduler); + registry.start(); + Thread.sleep(200); + String id = (String) registry.channelToChannelName(new DirectChannel()); + Thread.sleep(300); + assertNull(registry.channelNameToChannel(id)); + registry.stop(); + } + + public static class Foo extends AbstractReplyProducingMessageHandler { + + @Override + protected Object handleRequestMessage(Message requestMessage) { + assertThat(requestMessage.getHeaders().getReplyChannel(), + Matchers.anyOf(instanceOf(String.class), Matchers.nullValue())); + assertThat(requestMessage.getHeaders().getErrorChannel(), + Matchers.anyOf(instanceOf(String.class), Matchers.nullValue())); + if (requestMessage.getPayload().equals("bar")) { + throw new RuntimeException("intentional"); + } + return "echo:" + requestMessage.getPayload(); + } + + } + + public interface Gateway { + + String exchange(String foo); + + } + +} diff --git a/spring-integration-core/src/test/java/org/springframework/integration/config/xml/ControlBusTests.java b/spring-integration-core/src/test/java/org/springframework/integration/config/xml/ControlBusTests.java index a33a614b87..a5efb64f48 100644 --- a/spring-integration-core/src/test/java/org/springframework/integration/config/xml/ControlBusTests.java +++ b/spring-integration-core/src/test/java/org/springframework/integration/config/xml/ControlBusTests.java @@ -16,8 +16,8 @@ package org.springframework.integration.config.xml; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import java.util.Date; @@ -30,6 +30,9 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; import org.springframework.integration.Message; import org.springframework.integration.MessageChannel; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.channel.registry.DefaultHeaderChannelRegistry; +import org.springframework.integration.core.MessagingTemplate; import org.springframework.integration.core.PollableChannel; import org.springframework.integration.message.GenericMessage; import org.springframework.integration.support.MessageBuilder; @@ -52,6 +55,9 @@ public class ControlBusTests { @Autowired private PollableChannel output; + @Autowired + private DefaultHeaderChannelRegistry registry; + @Test public void testDefaultEvaluationContext() { Message message = MessageBuilder.withPayload("@service.convert('aardvark')+headers.foo").setHeader("foo", "bar").build(); @@ -71,6 +77,27 @@ public class ControlBusTests { assertNotNull(outputChannel.receive(1000)); } + @Test + public void testControlHeaderChannelReaper() throws InterruptedException { + MessagingTemplate messagingTemplate = new MessagingTemplate(); + messagingTemplate.convertAndSend(input, "@integrationHeaderChannelRegistry.size()"); + Message result = this.output.receive(0); + assertNotNull(result); + assertEquals(0, result.getPayload()); + this.registry.setReaperDelay(10); + this.registry.channelToChannelName(new DirectChannel()); + messagingTemplate.convertAndSend(input, "@integrationHeaderChannelRegistry.size()"); + result = this.output.receive(0); + assertNotNull(result); + assertEquals(1, result.getPayload()); + Thread.sleep(100); + messagingTemplate.convertAndSend(input, "@integrationHeaderChannelRegistry.runReaper()"); + messagingTemplate.convertAndSend(input, "@integrationHeaderChannelRegistry.size()"); + result = this.output.receive(0); + assertNotNull(result); + assertEquals(0, result.getPayload()); + this.registry.setReaperDelay(60000); + } public static class Service { @@ -78,12 +105,14 @@ public class ControlBusTests { public String convert(String input) { return "cat"; } + } public static class AdapterService { public Message receive() { return new GenericMessage(new Date().toString()); } + } } diff --git a/spring-integration-core/src/test/java/org/springframework/integration/config/xml/EnricherParserTests-context.xml b/spring-integration-core/src/test/java/org/springframework/integration/config/xml/EnricherParserTests-context.xml index b0c558e552..f62be84be9 100644 --- a/spring-integration-core/src/test/java/org/springframework/integration/config/xml/EnricherParserTests-context.xml +++ b/spring-integration-core/src/test/java/org/springframework/integration/config/xml/EnricherParserTests-context.xml @@ -13,11 +13,17 @@ + + + + + + diff --git a/spring-integration-core/src/test/java/org/springframework/integration/config/xml/EnricherParserTests.java b/spring-integration-core/src/test/java/org/springframework/integration/config/xml/EnricherParserTests.java index 18cb5ed44b..45dd1979a1 100644 --- a/spring-integration-core/src/test/java/org/springframework/integration/config/xml/EnricherParserTests.java +++ b/spring-integration-core/src/test/java/org/springframework/integration/config/xml/EnricherParserTests.java @@ -17,6 +17,7 @@ package org.springframework.integration.config.xml; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; @@ -79,6 +80,7 @@ public class EnricherParserTests { assertEquals(context.getBean("output"), accessor.getPropertyValue("outputChannel")); assertEquals(true, accessor.getPropertyValue("shouldClonePayload")); assertNull(accessor.getPropertyValue("requestPayloadExpression")); + assertNotNull(TestUtils.getPropertyValue(enricher, "gateway.beanFactory")); Map propertyExpressions = (Map) accessor.getPropertyValue("propertyExpressions"); for (Map.Entry e : propertyExpressions.entrySet()) { @@ -125,12 +127,15 @@ public class EnricherParserTests { @Test public void integrationTest() { SubscribableChannel requests = context.getBean("requests", SubscribableChannel.class); - requests.subscribe(new AbstractReplyProducingMessageHandler() { + class Foo extends AbstractReplyProducingMessageHandler { @Override protected Object handleRequestMessage(Message requestMessage) { return new Source("foo"); } - }); + }; + Foo foo = new Foo(); + foo.setOutputChannel(context.getBean("replies", MessageChannel.class)); + requests.subscribe(foo); Target original = new Target(); Message request = MessageBuilder.withPayload(original) .setHeader("sourceName", "test") diff --git a/spring-integration-jms/src/main/java/org/springframework/integration/jms/ChannelPublishingJmsMessageListener.java b/spring-integration-jms/src/main/java/org/springframework/integration/jms/ChannelPublishingJmsMessageListener.java index d88b55f555..d042b8c39a 100644 --- a/spring-integration-jms/src/main/java/org/springframework/integration/jms/ChannelPublishingJmsMessageListener.java +++ b/spring-integration-jms/src/main/java/org/springframework/integration/jms/ChannelPublishingJmsMessageListener.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2011 the original author or authors. + * Copyright 2002-2013 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +28,9 @@ import javax.jms.Session; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.BeanFactoryAware; import org.springframework.beans.factory.InitializingBean; import org.springframework.integration.Message; import org.springframework.integration.MessageChannel; @@ -47,18 +50,18 @@ import org.springframework.util.Assert; * Message and sends that Message to a channel. If the 'expectReply' value is * true, it will also wait for a Spring Integration reply Message * and convert that into a JMS reply. - * + * * @author Mark Fisher * @author Juergen Hoeller * @author Oleg Zhurakousky */ -public class ChannelPublishingJmsMessageListener - implements SessionAwareMessageListener, InitializingBean, TrackableComponent { - +public class ChannelPublishingJmsMessageListener + implements SessionAwareMessageListener, InitializingBean, TrackableComponent, BeanFactoryAware { + protected final Log logger = LogFactory.getLog(getClass()); - + private volatile boolean expectReply; - + private volatile MessageConverter messageConverter = new SimpleMessageConverter(); private volatile boolean extractRequestPayload = true; @@ -80,9 +83,11 @@ public class ChannelPublishingJmsMessageListener private volatile DestinationResolver destinationResolver = new DynamicDestinationResolver(); private volatile JmsHeaderMapper headerMapper = new DefaultJmsHeaderMapper(); - + private final GatewayDelegate gatewayDelegate = new GatewayDelegate(); + private volatile BeanFactory beanFactory; + /** * Specify whether a JMS reply Message is expected. */ @@ -93,39 +98,42 @@ public class ChannelPublishingJmsMessageListener public void setComponentName(String componentName){ this.gatewayDelegate.setComponentName(componentName); } - + public void setRequestChannel(MessageChannel requestChannel){ this.gatewayDelegate.setRequestChannel(requestChannel); } - + public void setReplyChannel(MessageChannel replyChannel){ this.gatewayDelegate.setReplyChannel(replyChannel); } - + public void setErrorChannel(MessageChannel errorChannel){ this.gatewayDelegate.setErrorChannel(errorChannel); } - + public void setRequestTimeout(long requestTimeout){ this.gatewayDelegate.setRequestTimeout(requestTimeout); } - + public void setReplyTimeout(long replyTimeout){ this.gatewayDelegate.setReplyTimeout(replyTimeout); } - + + @Override public void setShouldTrack(boolean shouldTrack) { this.gatewayDelegate.setShouldTrack(shouldTrack); } + @Override public String getComponentName() { return this.gatewayDelegate.getComponentName(); } + @Override public String getComponentType() { return this.gatewayDelegate.getComponentType(); } - + /** * Set the default reply destination to send reply messages to. This will * be applied in case of a request message that does not carry a @@ -189,7 +197,7 @@ public class ChannelPublishingJmsMessageListener * JMSMessageID from the request will be copied into the JMSCorrelationID of the reply * unless there is already a value in the JMSCorrelationID property of the newly created * reply Message in which case nothing will be copied. If the JMSCorrelationID of the - * request Message should be copied into the JMSCorrelationID of the reply Message + * request Message should be copied into the JMSCorrelationID of the reply Message * instead, then this value should be set to "JMSCorrelationID". * Any other value will be treated as a JMS String Property to be copied as-is * from the request Message into the reply Message with the same property name. @@ -200,7 +208,7 @@ public class ChannelPublishingJmsMessageListener /** * Specify whether explicit QoS should be enabled for replies - * (for timeToLive, priority, and deliveryMode settings). + * (for timeToLive, priority, and deliveryMode settings). */ public void setExplicitQosEnabledForReplies(boolean explicitQosEnabledForReplies) { this.explicitQosEnabledForReplies = explicitQosEnabledForReplies; @@ -224,7 +232,7 @@ public class ChannelPublishingJmsMessageListener * converting between JMS Messages and Spring Integration Messages. * If none is provided, a {@link SimpleMessageConverter} will * be used. - * + * * @param messageConverter */ public void setMessageConverter(MessageConverter messageConverter) { @@ -260,6 +268,12 @@ public class ChannelPublishingJmsMessageListener this.extractReplyPayload = extractReplyPayload; } + @Override + public void setBeanFactory(BeanFactory beanFactory) throws BeansException { + this.beanFactory = beanFactory; + } + + @Override public void onMessage(javax.jms.Message jmsMessage, Session session) throws JMSException { Object result = jmsMessage; if (this.extractRequestPayload) { @@ -268,10 +282,10 @@ public class ChannelPublishingJmsMessageListener logger.debug("converted JMS Message [" + jmsMessage + "] to integration Message payload [" + result + "]"); } } - + Map headers = headerMapper.toHeaders(jmsMessage); Message requestMessage = (result instanceof Message) ? - MessageBuilder.fromMessage((Message) result).copyHeaders(headers).build() : + MessageBuilder.fromMessage((Message) result).copyHeaders(headers).build() : MessageBuilder.withPayload(result).copyHeaders(headers).build(); if (!this.expectReply) { this.gatewayDelegate.send(requestMessage); @@ -305,18 +319,22 @@ public class ChannelPublishingJmsMessageListener } } + @Override public void afterPropertiesSet() { + if (this.beanFactory != null) { + this.gatewayDelegate.setBeanFactory(this.beanFactory); + } this.gatewayDelegate.afterPropertiesSet(); } - + protected void start(){ this.gatewayDelegate.start(); } - + protected void stop(){ this.gatewayDelegate.stop(); } - + private void copyCorrelationIdFromRequestToReply(javax.jms.Message requestMessage, javax.jms.Message replyMessage) throws JMSException { if (this.correlationKey != null) { if (this.correlationKey.equals("JMSCorrelationID")) { @@ -416,17 +434,20 @@ public class ChannelPublishingJmsMessageListener this.isTopic = isTopic; } } - + private class GatewayDelegate extends MessagingGatewaySupport { + @Override protected void send(Object request) { super.send(request); } - + + @Override protected Message sendAndReceiveMessage(Object request) { return super.sendAndReceiveMessage(request); } + @Override public String getComponentType() { if (expectReply) { return "jms:inbound-gateway"; diff --git a/spring-integration-jms/src/test/java/org/springframework/integration/jms/request_reply/GatewaySerializedReplyChannelTests-context.xml b/spring-integration-jms/src/test/java/org/springframework/integration/jms/request_reply/GatewaySerializedReplyChannelTests-context.xml new file mode 100644 index 0000000000..daeb4b956e --- /dev/null +++ b/spring-integration-jms/src/test/java/org/springframework/integration/jms/request_reply/GatewaySerializedReplyChannelTests-context.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spring-integration-jms/src/test/java/org/springframework/integration/jms/request_reply/GatewaySerializedReplyChannelTests.java b/spring-integration-jms/src/test/java/org/springframework/integration/jms/request_reply/GatewaySerializedReplyChannelTests.java new file mode 100644 index 0000000000..540c4abde3 --- /dev/null +++ b/spring-integration-jms/src/test/java/org/springframework/integration/jms/request_reply/GatewaySerializedReplyChannelTests.java @@ -0,0 +1,55 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.jms.request_reply; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.integration.Message; +import org.springframework.integration.MessageChannel; +import org.springframework.integration.core.PollableChannel; +import org.springframework.integration.message.GenericMessage; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +/** + * @author Gary Russell + * @since 3.0 + * + */ +@ContextConfiguration +@RunWith(SpringJUnit4ClassRunner.class) +public class GatewaySerializedReplyChannelTests { + + @Autowired + MessageChannel input; + + @Autowired + PollableChannel output; + + @Test + public void test() { + input.send(new GenericMessage("foo")); + Message reply = output.receive(0); + assertNotNull(reply); + assertEquals("echo:foo", reply.getPayload()); + } + +} diff --git a/src/reference/docbook/content-enrichment.xml b/src/reference/docbook/content-enrichment.xml index 6e8cf27f3d..657a5600c2 100644 --- a/src/reference/docbook/content-enrichment.xml +++ b/src/reference/docbook/content-enrichment.xml @@ -73,10 +73,10 @@ using generic <header> sub-elements where you would have to provide both header 'name' and 'value', you can use convenient sub-elements to set those values directly. - + - POJO Support + POJO Support @@ -121,7 +121,7 @@ ]]> - SpEL Support + SpEL Support In Spring Integration 2.0 we have introduced the convenience of the @@ -147,6 +147,51 @@ are bound to the SpEL Evaluation Context, giving you full access to the incoming Message. + + Header Channel Registry + + + Starting with Spring Integration 3.0, a new sub-element + <int:header-channels-to-string/> is available; it has no attributes. + This converts existing replyChannel and errorChannel + headers (when they are a + MessageChannel) to a String and stores the channel(s) in + a registry for later resolution when it is time to send a reply, or handle an error. + This is useful + for cases where the headers might be lost; for example when + serializing a message into a message store or when transporting the message + over JMS. If the header does not already exist, or it + is not a MessageChannel, no changes are made. + + + + Use of this functionality requires the presence of a HeaderChannelRegistry + bean. By default, the framework creates a DefaultHeaderChannelRegistry + with the default expiry (60 seconds). Channels + are removed from the registry after this time. To change this, simply define a bean + with id integrationHeaderChannelRegistry and configure the required delay using + a constructor argument (milliseconds). + + + + The HeaderChannelRegistry has a size() method to + determine the current size of the registry. The runReaper() method + cancels the current scheduled task and runs the reaper immediately; the task is + then scheduled to run again based on the current delay. These methods can be invoked + directly by getting a reference to the registry, or you can send a message with, for example, + the following content to a control bus: + + + + + + This sub-element is a convenience only, and is the equivalent of specifying: + + +]]> + For more examples for configuring header enrichers, see diff --git a/src/reference/docbook/message-store.xml b/src/reference/docbook/message-store.xml index 1a1a38297e..87976f6349 100644 --- a/src/reference/docbook/message-store.xml +++ b/src/reference/docbook/message-store.xml @@ -75,21 +75,21 @@ For example, if one of the headers contains an instance of some Spring Bean, upon deserialization you may end up with a different instance of that bean, which directly affects some of the implicit headers created by the framework (e.g., REPLY_CHANNEL or ERROR_CHANNEL). - Currently they are not serializable, but even if they were the deserialized channel would not represent the expected instance. - As a workaround we suggest to remove bean-ref headers via a <header-filter/> - before sending a message to an endpoint backed by a persistent MessageStore. - Also, we recommend using channel names instead of channel instances when setting those types of headers, - thus allowing it to be resolved in real time by the ChannelResolver. + Currently they are not serializable, but even if they were, the deserialized channel would not represent the expected instance. - Also avoid configuration of a message-flow like this: + Beginning with Spring Integration version 3.0, this issue can be resolved with a header enricher, + configured to replace these headers with a name after registering the channel with the HeaderChannelRegistry. + + + Also when configuring a message-flow like this: gateway -> queue-channel (backed by a persistent Message Store) -> service-activator - That gateway creates a Temporary Reply Channel in the background, and it will be lost by the time the - service-activator's poller reads from the queue, because it has been deserialized by another thread on the sending side. + That gateway creates a Temporary Reply Channel, and it will be lost by the time the + service-activator's poller reads from the queue. Again, you can use the header enricher to replace the headers with a + String representation. - Nevertheless we are constantly thinking about potential improvements to the framework, such as a way to provide some - robust default serialization strategy for messages in these cases. + For more information, refer to the . diff --git a/src/reference/docbook/whats-new.xml b/src/reference/docbook/whats-new.xml index c10f243959..08372f48d3 100644 --- a/src/reference/docbook/whats-new.xml +++ b/src/reference/docbook/whats-new.xml @@ -166,6 +166,16 @@ For more information see . +

+ Header Channel Registry + + It is now possible to instruct the framework to store reply and error channels + in a registry for later resolution. This is useful for cases where + the replyChannel or errorChannel might be lost; for example + when serializing + a message. See for more information. + +
From fa59b50bdc09c3a7deaae7c7ae5a64c48c69cce6 Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Fri, 15 Nov 2013 09:27:50 -0500 Subject: [PATCH 05/29] Fix Failing Test Producer starts before consumer (same phase) with a fast poller. Add negative phase to the consumer so he starts first. --- .../endpoint/ProducerAndConsumerAutoStartupTests-context.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spring-integration-core/src/test/java/org/springframework/integration/endpoint/ProducerAndConsumerAutoStartupTests-context.xml b/spring-integration-core/src/test/java/org/springframework/integration/endpoint/ProducerAndConsumerAutoStartupTests-context.xml index 1feb6b2cb0..7f873d3914 100644 --- a/spring-integration-core/src/test/java/org/springframework/integration/endpoint/ProducerAndConsumerAutoStartupTests-context.xml +++ b/spring-integration-core/src/test/java/org/springframework/integration/endpoint/ProducerAndConsumerAutoStartupTests-context.xml @@ -13,7 +13,7 @@ - + From 88de8117aa23f204fb20ce6528a5166a47f035ed Mon Sep 17 00:00:00 2001 From: Artem Bilan Date: Mon, 4 Nov 2013 17:30:46 +0200 Subject: [PATCH 06/29] INT-3140: Add `#xpath()` SpEL Function Support JIRA: https://jira.springsource.org/browse/INT-3140 * Introduce `XPathUtils` * Add `#xpath()` tests * Add documentation INT-3140: Polishing according PR comments INT-3140 Polishing - Doc and javadoc polishing - Fix some compiler warnings (not all xpath) --- .../AbstractIntegrationNamespaceHandler.java | 25 ++ .../integration/config/ChainParserTests.java | 3 +- .../integration/json/JsonPathTests.java | 15 +- .../SpelTransformerIntegrationTests.java | 2 +- .../integration/xml/xpath/XPathUtils.java | 131 ++++++++++ .../xml/xpath/XPathTests-context.xml | 39 +++ .../integration/xml/xpath/XPathTests.java | 227 ++++++++++++++++++ src/reference/docbook/spel.xml | 10 +- src/reference/docbook/transformer.xml | 7 + src/reference/docbook/whats-new.xml | 5 +- src/reference/docbook/xml.xml | 36 +++ 11 files changed, 484 insertions(+), 16 deletions(-) create mode 100644 spring-integration-xml/src/main/java/org/springframework/integration/xml/xpath/XPathUtils.java create mode 100644 spring-integration-xml/src/test/java/org/springframework/integration/xml/xpath/XPathTests-context.xml create mode 100644 spring-integration-xml/src/test/java/org/springframework/integration/xml/xpath/XPathTests.java diff --git a/spring-integration-core/src/main/java/org/springframework/integration/config/xml/AbstractIntegrationNamespaceHandler.java b/spring-integration-core/src/main/java/org/springframework/integration/config/xml/AbstractIntegrationNamespaceHandler.java index dd65c4337f..ba610212cf 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/config/xml/AbstractIntegrationNamespaceHandler.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/config/xml/AbstractIntegrationNamespaceHandler.java @@ -174,6 +174,31 @@ public abstract class AbstractIntegrationNamespaceHandler implements NamespaceHa } } + String xpathBeanName = "xpath"; + alreadyRegistered = false; + if (parserContext.getRegistry() instanceof ListableBeanFactory) { + alreadyRegistered = ((ListableBeanFactory) parserContext.getRegistry()).containsBean(xpathBeanName); + } + else { + alreadyRegistered = parserContext.getRegistry().isBeanNameInUse(xpathBeanName); + } + if (!alreadyRegistered) { + Class xpathClass = null; + try { + xpathClass = ClassUtils.forName(IntegrationNamespaceUtils.BASE_PACKAGE + ".xml.xpath.XPathUtils", + parserContext.getReaderContext().getBeanClassLoader()); + } + catch (ClassNotFoundException e) { + logger.debug("SpEL function '#xpath' isn't registered: there is no spring-integration-xml.jar on the classpath."); + } + + if (xpathClass != null) { + IntegrationNamespaceUtils.registerSpelFunctionBean(parserContext.getRegistry(), xpathBeanName, + IntegrationNamespaceUtils.BASE_PACKAGE + ".xml.xpath.XPathUtils", "evaluate"); + } + } + + this.doRegisterBuiltInBeans(parserContext); } diff --git a/spring-integration-core/src/test/java/org/springframework/integration/config/ChainParserTests.java b/spring-integration-core/src/test/java/org/springframework/integration/config/ChainParserTests.java index d07774fd09..bcc3c2fccb 100644 --- a/spring-integration-core/src/test/java/org/springframework/integration/config/ChainParserTests.java +++ b/spring-integration-core/src/test/java/org/springframework/integration/config/ChainParserTests.java @@ -301,6 +301,7 @@ public class ChainParserTests { final AtomicReference log = new AtomicReference(); when(logger.isWarnEnabled()).thenReturn(true); doAnswer(new Answer() { + @Override public Object answer(InvocationOnMock invocation) throws Throwable { log.set((String) invocation.getArguments()[0]); return null; @@ -394,7 +395,7 @@ public class ChainParserTests { assertTrue(this.beanFactory.containsBean("recipientListRouterChain$child.recipientListRouterWithinChain.handler")); MessageHandlerChain chain = this.beanFactory.getBean("headerEnricherChain.handler", MessageHandlerChain.class); - List handlers = TestUtils.getPropertyValue(chain, "handlers", List.class); + List handlers = TestUtils.getPropertyValue(chain, "handlers", List.class); assertTrue(handlers.get(0) instanceof MessageTransformingHandler); assertEquals("headerEnricherChain$child.headerEnricherWithinChain", TestUtils.getPropertyValue(handlers.get(0), "componentName")); diff --git a/spring-integration-core/src/test/java/org/springframework/integration/json/JsonPathTests.java b/spring-integration-core/src/test/java/org/springframework/integration/json/JsonPathTests.java index 7fae338604..8e8be5bdec 100644 --- a/spring-integration-core/src/test/java/org/springframework/integration/json/JsonPathTests.java +++ b/spring-integration-core/src/test/java/org/springframework/integration/json/JsonPathTests.java @@ -28,14 +28,11 @@ import java.io.IOException; import java.util.Map; import java.util.Scanner; +import org.hamcrest.Matchers; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; -import org.hamcrest.Matchers; - -import com.jayway.jsonpath.Criteria; -import com.jayway.jsonpath.Filter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -51,8 +48,12 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.test.context.support.AnnotationConfigContextLoader; +import com.jayway.jsonpath.Criteria; +import com.jayway.jsonpath.Filter; + /** * @author Artem Bilan + * @author Gary Russell * @since 3.0 */ @ContextConfiguration(classes = JsonPathTests.JsonPathTestsContextConfiguration.class, loader = AnnotationConfigContextLoader.class) @@ -69,7 +70,9 @@ public class JsonPathTests { public static void setUp() throws IOException { ClassPathResource jsonResource = new ClassPathResource("JsonPathTests.json", JsonPathTests.class); JSON_FILE = jsonResource.getFile(); - JSON = new Scanner(JSON_FILE).useDelimiter("\\Z").next(); + Scanner scanner = new Scanner(JSON_FILE); + JSON = scanner.useDelimiter("\\Z").next(); + scanner.close(); testMessage = new GenericMessage(JSON); } @@ -208,7 +211,7 @@ public class JsonPathTests { public static class JsonPathTestsContextConfiguration { @Bean - public Filter jsonPathFilter() { + public Filter jsonPathFilter() { return Filter.filter(Criteria.where("isbn").exists(true).and("category").ne("fiction")); } diff --git a/spring-integration-core/src/test/java/org/springframework/integration/transformer/SpelTransformerIntegrationTests.java b/spring-integration-core/src/test/java/org/springframework/integration/transformer/SpelTransformerIntegrationTests.java index 056624fc97..bba626a873 100644 --- a/spring-integration-core/src/test/java/org/springframework/integration/transformer/SpelTransformerIntegrationTests.java +++ b/spring-integration-core/src/test/java/org/springframework/integration/transformer/SpelTransformerIntegrationTests.java @@ -171,7 +171,7 @@ public class SpelTransformerIntegrationTests { @Override public Class[] getSpecificTargetClasses() { - return new Class[] {Foo.class}; + return new Class[] {Foo.class}; } @Override diff --git a/spring-integration-xml/src/main/java/org/springframework/integration/xml/xpath/XPathUtils.java b/spring-integration-xml/src/main/java/org/springframework/integration/xml/xpath/XPathUtils.java new file mode 100644 index 0000000000..d536879c86 --- /dev/null +++ b/spring-integration-xml/src/main/java/org/springframework/integration/xml/xpath/XPathUtils.java @@ -0,0 +1,131 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.xml.xpath; + + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.w3c.dom.Document; +import org.w3c.dom.Node; + +import org.springframework.integration.xml.DefaultXmlPayloadConverter; +import org.springframework.integration.xml.XmlPayloadConverter; +import org.springframework.util.Assert; +import org.springframework.xml.xpath.NodeMapper; +import org.springframework.xml.xpath.XPathException; +import org.springframework.xml.xpath.XPathExpression; +import org.springframework.xml.xpath.XPathExpressionFactory; + +/** + * Utility class for 'xpath' support. + * + * @author Artem Bilan + * @since 3.0 + */ +public final class XPathUtils { + + public static final String STRING = "string"; + + public static final String BOOLEAN = "boolean"; + + public static final String NUMBER = "number"; + + public static final String NODE = "node"; + + public static final String NODE_LIST = "node_list"; + + public static final String DOCUMENT_LIST = "document_list"; + + private static List RESULT_TYPES = Arrays.asList(STRING, BOOLEAN, NUMBER, NODE, NODE_LIST, DOCUMENT_LIST); + + private static XmlPayloadConverter converter = new DefaultXmlPayloadConverter(); + + private static DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); + + /** + * Utility method to evaluate an xpath on the provided object. + * Delegates evaluation to an {@link XPathExpression}. + * Note this method provides the {@code #xpath()} SpEL function. + * + * + * @param o the xml Object for evaluaton. + * @param xpath an 'xpath' expression String. + * @param resultArg an optional parameter to represent the result type of the xpath evaluation. + * Only one argument is allowed, which can be an instance of {@link org.springframework.xml.xpath.NodeMapper} or + * one of these String constants: "string", "boolean", "number", "node" or "node_list". + * @return the result of the xpath expression evaluation. + * @throws IllegalArgumentException - if the provided arguments aren't appropriate types or values; + * @throws MessagingException - if the provided object can't be converted to a {@link Node}; + * @throws XPathException - if the xpath expression can't be evaluated. + */ + @SuppressWarnings({"unchecked"}) + public static T evaluate(Object o, String xpath, Object... resultArg) { + Object resultType = null; + if (resultArg != null && resultArg.length > 0) { + Assert.isTrue(resultArg.length == 1, "'resultArg' can contains only one element."); + Assert.noNullElements(resultArg, "'resultArg' can't contains 'null' elements."); + resultType = resultArg[0]; + } + + XPathExpression expression = XPathExpressionFactory.createXPathExpression(xpath); + Node node = converter.convertToNode(o); + + if (resultType == null) { + return (T) expression.evaluateAsString(node); + } + else if (resultType instanceof NodeMapper) { + return (T) expression.evaluateAsObject(node, (NodeMapper) resultType); + } + else if (resultType instanceof String && RESULT_TYPES.contains(resultType)) { + String resType = (String) resultType; + if (DOCUMENT_LIST.equals(resType)) { + List nodeList = (List) XPathEvaluationType.NODE_LIST_RESULT.evaluateXPath(expression, node); + try { + DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder(); + List documents = new ArrayList(nodeList.size()); + for (Node n : nodeList) { + Document document = documentBuilder.newDocument(); + document.appendChild(document.importNode(n, true)); + documents.add(document); + } + return (T) documents; + } + catch (ParserConfigurationException e) { + throw new XPathException("Unable to create 'documentBuilder'.", e); + } + } + else { + XPathEvaluationType evaluationType = XPathEvaluationType.valueOf(resType.toUpperCase() + "_RESULT"); + return (T) evaluationType.evaluateXPath(expression, node); + } + } + else { + throw new IllegalArgumentException("'resultArg[0]' can be an instance of 'NodeMapper' " + + "or one of supported String constants: " + RESULT_TYPES); + } + } + + private XPathUtils() { + } + +} diff --git a/spring-integration-xml/src/test/java/org/springframework/integration/xml/xpath/XPathTests-context.xml b/spring-integration-xml/src/test/java/org/springframework/integration/xml/xpath/XPathTests-context.xml new file mode 100644 index 0000000000..961aac8eca --- /dev/null +++ b/spring-integration-xml/src/test/java/org/springframework/integration/xml/xpath/XPathTests-context.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spring-integration-xml/src/test/java/org/springframework/integration/xml/xpath/XPathTests.java b/spring-integration-xml/src/test/java/org/springframework/integration/xml/xpath/XPathTests.java new file mode 100644 index 0000000000..fab8b6fb39 --- /dev/null +++ b/spring-integration-xml/src/test/java/org/springframework/integration/xml/xpath/XPathTests.java @@ -0,0 +1,227 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.xml.xpath; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; +import static org.springframework.integration.xml.xpath.XPathUtils.evaluate; + +import java.util.Date; +import java.util.List; + +import org.hamcrest.Matchers; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.w3c.dom.DOMException; +import org.w3c.dom.Document; +import org.w3c.dom.Node; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.integration.Message; +import org.springframework.integration.MessageChannel; +import org.springframework.integration.MessagingException; +import org.springframework.integration.core.PollableChannel; +import org.springframework.integration.message.GenericMessage; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.integration.xml.source.StringSourceFactory; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.xml.xpath.NodeMapper; + +/** + * @author Artem Bilan + * @since 3.0 + */ +@ContextConfiguration +@RunWith(SpringJUnit4ClassRunner.class) +public class XPathTests { + + private static final String XML = ""; + + @Autowired + private PollableChannel channelA; + + @Autowired + private PollableChannel channelB; + + @Autowired + private PollableChannel channelZ; + + @Autowired + private MessageChannel xpathTransformerInput; + + @Autowired + private MessageChannel xpathFilterInput; + + @Autowired + private MessageChannel xpathSplitterInput; + + @Autowired + private MessageChannel xpathRouterInput; + + @Test + @SuppressWarnings("unchecked") + public void testXPathUtils() { + Object result = evaluate(XML, "/parent/child/@name"); + assertEquals("test", result); + + result = evaluate(XML, "/parent/child/@name", "string"); + assertEquals("test", result); + + result = evaluate(XML, "/parent/child/@age", "number"); + assertEquals((double) 42, result); + + result = evaluate(XML, "/parent/child/@married = 'true'", "boolean"); + assertEquals(Boolean.TRUE, result); + + result = evaluate(XML, "/parent/child", "node"); + assertThat(result, Matchers.instanceOf(Node.class)); + Node node = (Node) result; + assertEquals("child", node.getLocalName()); + assertEquals("test", node.getAttributes().getNamedItem("name").getTextContent()); + assertEquals("42", node.getAttributes().getNamedItem("age").getTextContent()); + assertEquals("true", node.getAttributes().getNamedItem("married").getTextContent()); + + result = evaluate("", "/parent/child", "node_list"); + assertThat(result, Matchers.instanceOf(List.class)); + List nodeList = (List) result; + assertEquals(2, nodeList.size()); + Node node1 = nodeList.get(0); + Node node2 = nodeList.get(1); + assertEquals("child", node1.getLocalName()); + assertEquals("foo", node1.getAttributes().getNamedItem("name").getTextContent()); + assertEquals("child", node2.getLocalName()); + assertEquals("bar", node2.getAttributes().getNamedItem("name").getTextContent()); + + result = evaluate("", "/parent/child", "document_list"); + assertThat(result, Matchers.instanceOf(List.class)); + List documentList = (List) result; + assertEquals(2, documentList.size()); + Node document1 = documentList.get(0); + Node document2 = documentList.get(1); + assertEquals("child", document1.getFirstChild().getLocalName()); + assertEquals("foo", document1.getFirstChild().getAttributes().getNamedItem("name").getTextContent()); + assertEquals("child", document2.getFirstChild().getLocalName()); + assertEquals("bar", document2.getFirstChild().getAttributes().getNamedItem("name").getTextContent()); + + result = evaluate(XML, "/parent/child/@name", new TestNodeMapper()); + assertEquals("test-mapped", result); + + try { + evaluate(new Date(), "/parent/child"); + fail("MessagingException expected."); + } + catch (Exception e) { + assertThat(e, Matchers.instanceOf(MessagingException.class)); + assertThat(e.getMessage(), Matchers.containsString("unsupported payload type")); + } + + try { + evaluate(XML, "/parent/child", "string", "number"); + fail("MessagingException expected."); + } + catch (Exception e) { + assertThat(e, Matchers.instanceOf(IllegalArgumentException.class)); + assertEquals("'resultArg' can contains only one element.", e.getMessage()); + } + + try { + evaluate(XML, "/parent/child", "foo"); + fail("MessagingException expected."); + } + catch (Exception e) { + assertThat(e, Matchers.instanceOf(IllegalArgumentException.class)); + assertEquals("'resultArg[0]' can be an instance of 'NodeMapper' or " + + "one of supported String constants: [string, boolean, number, node, node_list, document_list]", e.getMessage()); + } + + } + + @Test + public void testInt3140Transformer() { + Message message = MessageBuilder.withPayload("") + .setHeader("xpath", "/person/@age") + .build(); + + this.xpathTransformerInput.send(message); + + Message receive = this.channelA.receive(1000); + assertNotNull(receive); + assertEquals("42-mapped", receive.getPayload()); + } + + @Test + public void testInt3140Filter() { + this.xpathFilterInput.send(new GenericMessage("outputOne")); + this.xpathFilterInput.send(new GenericMessage("outputOne")); + + Message receive = this.channelA.receive(1000); + assertNotNull(receive); + assertEquals("outputOne", receive.getPayload()); + + receive = this.channelZ.receive(1000); + assertNotNull(receive); + assertEquals("outputOne", receive.getPayload()); + } + + @Test + public void testInt3140Splitter() { + StringSourceFactory stringSourceFactory = new StringSourceFactory(); + this.xpathSplitterInput.send(new GenericMessage("book1book2")); + + Message receive = this.channelA.receive(1000); + assertNotNull(receive); + assertThat(stringSourceFactory.createSource(receive.getPayload()).toString(), Matchers.containsString("book1")); + + receive = this.channelA.receive(1000); + assertNotNull(receive); + assertThat(stringSourceFactory.createSource(receive.getPayload()).toString(), Matchers.containsString("book2")); + } + + + @Test + public void testInt3140Router() { + this.xpathRouterInput.send(new GenericMessage("A")); + this.xpathRouterInput.send(new GenericMessage("B")); + this.xpathRouterInput.send(new GenericMessage("X")); + + Message receive = this.channelA.receive(1000); + assertNotNull(receive); + assertEquals("A", receive.getPayload()); + + receive = this.channelB.receive(1000); + assertNotNull(receive); + assertEquals("B", receive.getPayload()); + + receive = this.channelZ.receive(1000); + assertNotNull(receive); + assertEquals("X", receive.getPayload()); + } + + + public static class TestNodeMapper implements NodeMapper { + + @Override + public String mapNode(Node node, int nodeNum) throws DOMException { + return node.getTextContent() + "-mapped"; + } + + } + +} diff --git a/src/reference/docbook/spel.xml b/src/reference/docbook/spel.xml index 157ea4ca0b..6518653bb5 100644 --- a/src/reference/docbook/spel.xml +++ b/src/reference/docbook/spel.xml @@ -176,13 +176,11 @@ For more information regarding JSON see 'JSON Transformers' in . + + #xpath - to evaluate an 'xpath' on some provided object. + For more information regarding xml and xpath see . + diff --git a/src/reference/docbook/transformer.xml b/src/reference/docbook/transformer.xml index 495664b267..9caf015cc2 100644 --- a/src/reference/docbook/transformer.xml +++ b/src/reference/docbook/transformer.xml @@ -361,6 +361,13 @@ public class Foo { In addition to JSON Transformers, Spring Integration provides a built-in #jsonPath SpEL function for use in expressions. For more information see . + + #xpath SpEL Function + + + Since version 3.0, Spring Integration also provides a built-in #xpath + SpEL function for use in expressions. For more information see . +
diff --git a/src/reference/docbook/whats-new.xml b/src/reference/docbook/whats-new.xml index 08372f48d3..2cbea47e51 100644 --- a/src/reference/docbook/whats-new.xml +++ b/src/reference/docbook/whats-new.xml @@ -117,8 +117,9 @@ SpEL Functions Support To customize the SpEL EvaluationContext with static - Method functions the new <spel-function/> - component is introduced. For more information see . + Method functions, the new <spel-function/> + component is introduced. Two built-in functions are also provided (#jsonPath + and #xpath). For more information see .
diff --git a/src/reference/docbook/xml.xml b/src/reference/docbook/xml.xml index 427d4c0723..5ee55ef17a 100644 --- a/src/reference/docbook/xml.xml +++ b/src/reference/docbook/xml.xml @@ -34,6 +34,9 @@ XPath Filter + + #xpath SpEL Function + Validating Filter @@ -1253,6 +1256,39 @@
+
+ #xpath SpEL Function + + Spring Integration, since version 3.0, provides the #xpath + built-in SpEL function, which invokes the static method XPathUtils.evaluate(...). + This method delegates to an org.springframework.xml.xpath.XPathExpression. + The following shows some usage examples: + + + + + + + + + +]]> + #xpath also supports a third optional parameter for converting the result of the xpath evaluation. + It can be + one of the String constants 'string', 'boolean', 'number', + 'node', 'node_list' and 'document_list' or an + org.springframework.xml.xpath.NodeMapper instance. + By default the #xpath SpEL function returns a String representation of the xpath evaluation. + + + To enable the #xpath SpEL function, simply add the spring-integration-xml.jar + to the CLASSPATH; there is no need to declare any component(s) from the Spring Integration Xml Namespace. + + + For more information see . + +
+
XML Validating Filter From 59d6f7dfc1da8c26e8bd0121d8f2ee7cb796c4a2 Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Tue, 19 Nov 2013 12:56:07 +0200 Subject: [PATCH 07/29] INT-3088 (S)FTP Outbound Gateway - PUT and MPUT - Core support in file module - FTP Parser and Test https://jira.springsource.org/browse/INT-3088 INT-3088 Add SFTP Support for PUT, MPUT INT-3088 Polishing - PR Comments INT-3088 Docbook For PUT/MPUT --- ...stractRemoteFileOutboundGatewayParser.java | 39 ++-- .../file/config/FileParserUtils.java | 91 ++++++++ ...emoteFileOutboundChannelAdapterParser.java | 72 +----- ...actPersistentAcceptOnceFileListFilter.java | 4 +- .../file/remote/RemoteFileOperations.java | 17 +- .../file/remote/RemoteFileTemplate.java | 26 ++- .../file/remote/SessionCallback.java | 2 +- .../AbstractRemoteFileOutboundGateway.java | 148 ++++++++++-- .../FileTransferringMessageHandler.java | 5 + .../config/spring-integration-file-3.0.xsd | 72 ++++++ .../RemoteFileOutboundGatewayTests.java | 134 +++++++++++ .../ftp/gateway/FtpOutboundGateway.java | 6 + .../ftp/config/spring-integration-ftp-3.0.xsd | 120 ++++------ .../integration/ftp/TesFtpServer.java | 10 +- .../FtpOutboundGatewayParserTests-context.xml | 22 ++ .../config/FtpOutboundGatewayParserTests.java | 37 ++- .../FtpServerOutboundTests-context.xml | 36 +++ .../ftp/outbound/FtpServerOutboundTests.java | 74 +++++- .../sftp/gateway/SftpOutboundGateway.java | 11 +- .../config/spring-integration-sftp-3.0.xsd | 124 ++++------ ...ChannelAdapterParserTests-context-fail.xml | 2 +- .../OutboundChannelAdapterParserTests.java | 13 +- ...SftpOutboundGatewayParserTests-context.xml | 26 +++ .../SftpOutboundGatewayParserTests.java | 37 ++- .../SftpServerOutboundTests-context.xml | 36 +++ .../outbound/SftpServerOutboundTests.java | 217 ++++++++++++------ src/reference/docbook/ftp.xml | 48 ++++ src/reference/docbook/sftp.xml | 49 ++++ src/reference/docbook/whats-new.xml | 30 ++- 29 files changed, 1163 insertions(+), 345 deletions(-) create mode 100644 spring-integration-file/src/main/java/org/springframework/integration/file/config/FileParserUtils.java diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileOutboundGatewayParser.java b/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileOutboundGatewayParser.java index 8fc19e0bc2..2e90df8dd1 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileOutboundGatewayParser.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileOutboundGatewayParser.java @@ -23,6 +23,8 @@ import org.springframework.beans.factory.xml.ParserContext; import org.springframework.integration.config.ExpressionFactoryBean; import org.springframework.integration.config.xml.AbstractConsumerEndpointParser; import org.springframework.integration.config.xml.IntegrationNamespaceUtils; +import org.springframework.integration.file.filters.RegexPatternFileListFilter; +import org.springframework.integration.file.filters.SimplePatternFileListFilter; import org.springframework.util.StringUtils; /** @@ -42,18 +44,20 @@ public abstract class AbstractRemoteFileOutboundGatewayParser extends AbstractCo @Override protected BeanDefinitionBuilder parseHandler(Element element, ParserContext parserContext) { + + BeanDefinition templateDefinition = FileParserUtils.parseRemoteFileTemplate(element, parserContext, false); + BeanDefinitionBuilder builder = BeanDefinitionBuilder.genericBeanDefinition(getGatewayClassName()); - builder.addConstructorArgReference(element.getAttribute("session-factory")); + builder.addConstructorArgValue(templateDefinition); builder.addConstructorArgValue(element.getAttribute("command")); builder.addConstructorArgValue(element.getAttribute(EXPRESSION_ATTRIBUTE)); IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "command-options", "options"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "temporary-file-suffix"); IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "reply-timeout", "sendTimeout"); IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, element, "reply-channel", "outputChannel"); - this.configureFilter(builder, element, parserContext); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "remote-file-separator"); + this.configureFilter(builder, element, parserContext, "filter", "filename", "filter"); + this.configureFilter(builder, element, parserContext, "mput-filter", "mput", "mputFilter"); BeanDefinition localDirExpressionDef = IntegrationNamespaceUtils .createExpressionDefinitionFromValueOrExpression("local-directory", "local-directory-expression", @@ -74,10 +78,11 @@ public abstract class AbstractRemoteFileOutboundGatewayParser extends AbstractCo return builder; } - protected void configureFilter(BeanDefinitionBuilder builder, Element element, ParserContext parserContext) { - String filter = element.getAttribute("filter"); - String fileNamePattern = element.getAttribute("filename-pattern"); - String fileNameRegex = element.getAttribute("filename-regex"); + protected void configureFilter(BeanDefinitionBuilder builder, Element element, ParserContext parserContext, + String filterAttribute, String patternPrefix, String propertyName) { + String filter = element.getAttribute(filterAttribute); + String fileNamePattern = element.getAttribute(patternPrefix + "-pattern"); + String fileNameRegex = element.getAttribute(patternPrefix + "-regex"); boolean hasFilter = StringUtils.hasText(filter); boolean hasFileNamePattern = StringUtils.hasText(fileNamePattern); boolean hasFileNameRegex = StringUtils.hasText(fileNameRegex); @@ -85,23 +90,27 @@ public abstract class AbstractRemoteFileOutboundGatewayParser extends AbstractCo count += hasFileNamePattern ? 1 : 0; count += hasFileNameRegex ? 1 : 0; if (count > 1) { - parserContext.getReaderContext().error("at most one of 'filename-pattern', " + - "'filename-regex', or 'filter' is allowed on remote file inbound adapter", element); + parserContext.getReaderContext().error("at most one of '" + patternPrefix + "-pattern', " + + "'" + patternPrefix + "-regex', or '" + filterAttribute + "' is allowed on a remote file outbound gateway", element); } else if (hasFilter) { - builder.addPropertyReference("filter", filter); + builder.addPropertyReference(propertyName, filter); } else if (hasFileNamePattern) { BeanDefinitionBuilder filterBuilder = BeanDefinitionBuilder.genericBeanDefinition( - this.getSimplePatternFileListFilterClassName()); + "filter".equals(filterAttribute) ? + this.getSimplePatternFileListFilterClassName() : + SimplePatternFileListFilter.class.getName()); filterBuilder.addConstructorArgValue(fileNamePattern); - builder.addPropertyValue("filter", filterBuilder.getBeanDefinition()); + builder.addPropertyValue(propertyName, filterBuilder.getBeanDefinition()); } else if (hasFileNameRegex) { BeanDefinitionBuilder filterBuilder = BeanDefinitionBuilder.genericBeanDefinition( - this.getRegexPatternFileListFilterClassName()); + "filter".equals(filterAttribute) ? + this.getRegexPatternFileListFilterClassName() : + RegexPatternFileListFilter.class.getName()); filterBuilder.addConstructorArgValue(fileNameRegex); - builder.addPropertyValue("filter", filterBuilder.getBeanDefinition()); + builder.addPropertyValue(propertyName, filterBuilder.getBeanDefinition()); } } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/config/FileParserUtils.java b/spring-integration-file/src/main/java/org/springframework/integration/file/config/FileParserUtils.java new file mode 100644 index 0000000000..e9ea713af8 --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/config/FileParserUtils.java @@ -0,0 +1,91 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.file.config; + +import org.w3c.dom.Element; + +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.xml.ParserContext; +import org.springframework.integration.config.xml.IntegrationNamespaceUtils; +import org.springframework.integration.file.DefaultFileNameGenerator; +import org.springframework.integration.file.remote.RemoteFileTemplate; +import org.springframework.util.StringUtils; + +/** + * @author Oleg Zhurakousky + * @author Mark Fisher + * @author David Turanski + * @author Gary Russell + * @since 3.0 + * + */ +public final class FileParserUtils { + + private FileParserUtils() { + } + + public static BeanDefinition parseRemoteFileTemplate(Element element, ParserContext parserContext, + boolean atLeastOneRemoteDirectoryAttributeRequired) { + BeanDefinitionBuilder templateBuilder = BeanDefinitionBuilder.genericBeanDefinition(RemoteFileTemplate.class); + + templateBuilder.addConstructorArgReference(element.getAttribute("session-factory")); + // configure MessageHandler properties + + IntegrationNamespaceUtils.setValueIfAttributeDefined(templateBuilder, element, "temporary-file-suffix"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(templateBuilder, element, "use-temporary-file-name"); + + IntegrationNamespaceUtils.setValueIfAttributeDefined(templateBuilder, element, "auto-create-directory"); + + BeanDefinition expressionDef = + IntegrationNamespaceUtils.createExpressionDefinitionFromValueOrExpression("remote-directory", + "remote-directory-expression", parserContext, element, atLeastOneRemoteDirectoryAttributeRequired); + if (expressionDef != null) { + templateBuilder.addPropertyValue("remoteDirectoryExpression", expressionDef); + } + expressionDef = IntegrationNamespaceUtils.createExpressionDefinitionFromValueOrExpression("temporary-remote-directory", + "temporary-remote-directory-expression", parserContext, element, false); + if (expressionDef != null) { + templateBuilder.addPropertyValue("temporaryRemoteDirectoryExpression", expressionDef); + } + + // configure remote FileNameGenerator + String remoteFileNameGenerator = element.getAttribute("remote-filename-generator"); + String remoteFileNameGeneratorExpression = element.getAttribute("remote-filename-generator-expression"); + boolean hasRemoteFileNameGenerator = StringUtils.hasText(remoteFileNameGenerator); + boolean hasRemoteFileNameGeneratorExpression = StringUtils.hasText(remoteFileNameGeneratorExpression); + if (hasRemoteFileNameGenerator || hasRemoteFileNameGeneratorExpression) { + if (hasRemoteFileNameGenerator && hasRemoteFileNameGeneratorExpression) { + parserContext.getReaderContext().error( + "at most one of 'remote-filename-generator-expression' or 'remote-filename-generator' " + + "is allowed on a remote file outbound adapter", element); + } + if (hasRemoteFileNameGenerator) { + templateBuilder.addPropertyReference("fileNameGenerator", remoteFileNameGenerator); + } + else { + BeanDefinitionBuilder fileNameGeneratorBuilder = BeanDefinitionBuilder + .genericBeanDefinition(DefaultFileNameGenerator.class); + fileNameGeneratorBuilder.addPropertyValue("expression", remoteFileNameGeneratorExpression); + templateBuilder.addPropertyValue("fileNameGenerator", fileNameGeneratorBuilder.getBeanDefinition()); + } + } + IntegrationNamespaceUtils.setValueIfAttributeDefined(templateBuilder, element, "charset"); + templateBuilder.addPropertyValue("remoteFileSeparator", element.getAttribute("remote-file-separator")); + return templateBuilder.getBeanDefinition(); + } + +} diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/config/RemoteFileOutboundChannelAdapterParser.java b/spring-integration-file/src/main/java/org/springframework/integration/file/config/RemoteFileOutboundChannelAdapterParser.java index 919c79114a..2d4faa9ea6 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/config/RemoteFileOutboundChannelAdapterParser.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/config/RemoteFileOutboundChannelAdapterParser.java @@ -18,19 +18,12 @@ package org.springframework.integration.file.config; import org.w3c.dom.Element; -import org.springframework.beans.factory.BeanDefinitionStoreException; import org.springframework.beans.factory.config.BeanDefinition; import org.springframework.beans.factory.support.AbstractBeanDefinition; import org.springframework.beans.factory.support.BeanDefinitionBuilder; -import org.springframework.beans.factory.support.RootBeanDefinition; import org.springframework.beans.factory.xml.ParserContext; -import org.springframework.expression.common.LiteralExpression; -import org.springframework.integration.config.ExpressionFactoryBean; import org.springframework.integration.config.xml.AbstractOutboundChannelAdapterParser; -import org.springframework.integration.config.xml.IntegrationNamespaceUtils; -import org.springframework.integration.file.DefaultFileNameGenerator; import org.springframework.integration.file.remote.handler.FileTransferringMessageHandler; -import org.springframework.util.StringUtils; /** * @author Oleg Zhurakousky @@ -45,71 +38,10 @@ public class RemoteFileOutboundChannelAdapterParser extends AbstractOutboundChan protected AbstractBeanDefinition parseConsumer(Element element, ParserContext parserContext) { BeanDefinitionBuilder handlerBuilder = BeanDefinitionBuilder.genericBeanDefinition(FileTransferringMessageHandler.class); - handlerBuilder.addConstructorArgReference(element.getAttribute("session-factory")); - // configure MessageHandler properties + BeanDefinition templateDefinition = FileParserUtils.parseRemoteFileTemplate(element, parserContext, true); - IntegrationNamespaceUtils.setValueIfAttributeDefined(handlerBuilder, element, "temporary-file-suffix"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(handlerBuilder, element, "use-temporary-file-name"); - - IntegrationNamespaceUtils.setValueIfAttributeDefined(handlerBuilder, element, "auto-create-directory"); - - this.configureRemoteDirectories(element, handlerBuilder); - - // configure remote FileNameGenerator - String remoteFileNameGenerator = element.getAttribute("remote-filename-generator"); - String remoteFileNameGeneratorExpression = element.getAttribute("remote-filename-generator-expression"); - boolean hasRemoteFileNameGenerator = StringUtils.hasText(remoteFileNameGenerator); - boolean hasRemoteFileNameGeneratorExpression = StringUtils.hasText(remoteFileNameGeneratorExpression); - if (hasRemoteFileNameGenerator || hasRemoteFileNameGeneratorExpression) { - if (hasRemoteFileNameGenerator && hasRemoteFileNameGeneratorExpression) { - throw new BeanDefinitionStoreException("at most one of 'remote-filename-generator-expression' or 'remote-filename-generator' " + - "is allowed on a remote file outbound adapter"); - } - if (hasRemoteFileNameGenerator) { - handlerBuilder.addPropertyReference("fileNameGenerator", remoteFileNameGenerator); - } - else { - BeanDefinitionBuilder fileNameGeneratorBuilder = BeanDefinitionBuilder.genericBeanDefinition(DefaultFileNameGenerator.class); - fileNameGeneratorBuilder.addPropertyValue("expression", remoteFileNameGeneratorExpression); - handlerBuilder.addPropertyValue("fileNameGenerator", fileNameGeneratorBuilder.getBeanDefinition()); - } - } - IntegrationNamespaceUtils.setValueIfAttributeDefined(handlerBuilder, element, "charset"); - handlerBuilder.addPropertyValue("remoteFileSeparator", element.getAttribute("remote-file-separator")); + handlerBuilder.addConstructorArgValue(templateDefinition); return handlerBuilder.getBeanDefinition(); } - private void configureRemoteDirectories(Element element, BeanDefinitionBuilder handlerBuilder){ - this.doConfigureRemoteDirectory(element, handlerBuilder, "remote-directory", "remote-directory-expression", "remoteDirectoryExpression", true); - this.doConfigureRemoteDirectory(element, handlerBuilder, "temporary-remote-directory", "temporary-remote-directory-expression", "temporaryRemoteDirectoryExpression", false); - } - - private void doConfigureRemoteDirectory(Element element, BeanDefinitionBuilder handlerBuilder, - String directoryAttribute, String directoryExpressionAttribute, - String directoryExpressionPropertyName, boolean atLeastOneRequired){ - String remoteDirectory = element.getAttribute(directoryAttribute); - String remoteDirectoryExpression = element.getAttribute(directoryExpressionAttribute); - boolean hasRemoteDirectory = StringUtils.hasText(remoteDirectory); - boolean hasRemoteDirectoryExpression = StringUtils.hasText(remoteDirectoryExpression); - if (atLeastOneRequired){ - if (!(hasRemoteDirectory ^ hasRemoteDirectoryExpression)) { - throw new BeanDefinitionStoreException("exactly one of '" + directoryAttribute + "' or '" + directoryExpressionAttribute + "' " + - "is required on a remote file outbound adapter"); - } - } - - BeanDefinition remoteDirectoryExpressionDefinition = null; - if (hasRemoteDirectory) { - remoteDirectoryExpressionDefinition = new RootBeanDefinition(LiteralExpression.class); - remoteDirectoryExpressionDefinition.getConstructorArgumentValues().addGenericArgumentValue(remoteDirectory); - } - else if (hasRemoteDirectoryExpression) { - remoteDirectoryExpressionDefinition = new RootBeanDefinition(ExpressionFactoryBean.class); - remoteDirectoryExpressionDefinition.getConstructorArgumentValues().addGenericArgumentValue(remoteDirectoryExpression); - } - if (remoteDirectoryExpressionDefinition != null){ - handlerBuilder.addPropertyValue(directoryExpressionPropertyName, remoteDirectoryExpressionDefinition); - } - } - } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java index 0f752473f5..3a44133ca7 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java @@ -69,8 +69,8 @@ public abstract class AbstractPersistentAcceptOnceFileListFilter extends Abst * Override this method if you wish to use something other than the * modified timestamp to determine equality. * @param file The file. - * @param value The current value for the key in the store - * @return + * @param value The current value for the key in the store. + * @return true if equal. */ protected boolean isEqual(F file, String value) { return Long.valueOf(value).longValue() == this.modified(file); diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java index 3f5ca23732..56c49c82e4 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java @@ -29,11 +29,24 @@ public interface RemoteFileOperations { /** * Send a file to a remote server, based on information in a message. * - * @param message The message + * @param message The message. + * @return The remote path, or null if no local file was found. * @throws Exception */ - void send(Message message); + String send(Message message); + /** + * Send a file to a remote server, based on information in a message. + * The subDirectory is appended to the remote directory evaluated from + * the message. + * + * @param message The message. + * @param subDirectory The sub directory. + * @return The remote path, or null if no local file was found. + * @throws Exception + */ + + String send(Message message, String subDirectory); /** * Retrieve a remote file as an InputStream, based on information in a message. * diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java index 02e3918ee2..b11f503e36 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java @@ -99,6 +99,10 @@ public class RemoteFileTemplate implements RemoteFileOperations, Initializ this.remoteFileSeparator = remoteFileSeparator; } + public final String getRemoteFileSeparator() { + return remoteFileSeparator; + } + public void setRemoteDirectoryExpression(Expression remoteDirectoryExpression) { Assert.notNull(remoteDirectoryExpression, "remoteDirectoryExpression must not be null"); this.directoryExpressionProcessor = new ExpressionEvaluatingMessageProcessor(remoteDirectoryExpression, String.class); @@ -173,18 +177,32 @@ public class RemoteFileTemplate implements RemoteFileOperations, Initializ } @Override - public void send(final Message message) { + public String send(final Message message) { + return this.send(message, null); + } + + @Override + public String send(final Message message, final String subDirectory) { Assert.notNull(this.directoryExpressionProcessor, "'remoteDirectoryExpression' is required"); final StreamHolder inputStreamHolder = this.payloadToInputStream(message); if (inputStreamHolder != null) { - this.execute(new SessionCallbackWithoutResult() { + return this.execute(new SessionCallback() { @Override - public void doInSessionWithoutResult(Session session) throws IOException { + public String doInSession(Session session) throws IOException { String fileName = inputStreamHolder.getName(); try { String remoteDirectory = RemoteFileTemplate.this.directoryExpressionProcessor .processMessage(message); + remoteDirectory = RemoteFileTemplate.this.normalizeDirectoryPath(remoteDirectory); + if (StringUtils.hasText(subDirectory)) { + if (subDirectory.startsWith(RemoteFileTemplate.this.remoteFileSeparator)) { + remoteDirectory += subDirectory.substring(1); + } + else { + remoteDirectory += RemoteFileTemplate.this.normalizeDirectoryPath(subDirectory); + } + } String temporaryRemoteDirectory = remoteDirectory; if (RemoteFileTemplate.this.temporaryDirectoryExpressionProcessor != null) { temporaryRemoteDirectory = RemoteFileTemplate.this.temporaryDirectoryExpressionProcessor @@ -193,6 +211,7 @@ public class RemoteFileTemplate implements RemoteFileOperations, Initializ fileName = RemoteFileTemplate.this.fileNameGenerator.generateFileName(message); RemoteFileTemplate.this.sendFileToRemoteDirectory(inputStreamHolder.getStream(), temporaryRemoteDirectory, remoteDirectory, fileName, session); + return remoteDirectory + fileName; } catch (FileNotFoundException e) { throw new MessageDeliveryException(message, "File [" + inputStreamHolder.getName() @@ -215,6 +234,7 @@ public class RemoteFileTemplate implements RemoteFileOperations, Initializ if (logger.isWarnEnabled()) { logger.warn("File " + message.getPayload() + " does not exist"); } + return null; } } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/SessionCallback.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/SessionCallback.java index 2f0fc06277..1970bb2a6d 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/SessionCallback.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/SessionCallback.java @@ -20,7 +20,7 @@ import java.io.IOException; import org.springframework.integration.file.remote.session.Session; /** - * Callback invoked by {@code RemoteFileOperations.execute()) - allows multiple operations + * Callback invoked by {@code RemoteFileOperations.execute()} - allows multiple operations * on a session. * * @author Gary Russell diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java index e1ece85a17..024a4c46eb 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java @@ -92,7 +92,17 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply /** * Move (rename) a remote file. */ - MV("mv"); + MV("mv"), + + /** + * Put a local file to the remote system. + */ + PUT("put"), + + /** + * Put multiple local files to the remote system. + */ + MPUT("mput"); private String command; @@ -188,19 +198,20 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply protected volatile Set