diff --git a/build.gradle b/build.gradle index e1df0e6656..c64cb964c2 100644 --- a/build.gradle +++ b/build.gradle @@ -258,6 +258,9 @@ project('spring-integration-file') { compile project(":spring-integration-core") compile "org.springframework:spring-context:$springVersion" compile "commons-io:commons-io:$commonsIoVersion" + testCompile project(":spring-integration-redis") + testCompile project(":spring-integration-redis").sourceSets.test.output + testCompile "com.lambdaworks:lettuce:$lettuceVersion" } } diff --git a/spring-integration-core/src/main/java/org/springframework/integration/metadata/ConcurrentMetadataStore.java b/spring-integration-core/src/main/java/org/springframework/integration/metadata/ConcurrentMetadataStore.java new file mode 100644 index 0000000000..e7b9c79a6d --- /dev/null +++ b/spring-integration-core/src/main/java/org/springframework/integration/metadata/ConcurrentMetadataStore.java @@ -0,0 +1,49 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.metadata; + + + +/** + * Supports atomic updates to values in the store. + * + * @author Gary Russell + * @since 4.0 + * + */ +public interface ConcurrentMetadataStore extends MetadataStore { + + /** + * Atomically insert the key into the store. + * + * @param key The key. + * @param value The value. + * @return null if successful, the old value otherwise. + */ + String putIfAbsent(String key, String value); + + /** + * Atomically replace the value for the key in the store if the old + * value matches the oldValue argument. + * + * @param key The key. + * @param oldValue The old value. + * @param newValue The new value. + * @return true if successful. + */ + boolean replace(String key, String oldValue, String newValue); + +} diff --git a/spring-integration-core/src/main/java/org/springframework/integration/metadata/PropertiesPersistingMetadataStore.java b/spring-integration-core/src/main/java/org/springframework/integration/metadata/PropertiesPersistingMetadataStore.java index b6599cef0d..605f334048 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/metadata/PropertiesPersistingMetadataStore.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/metadata/PropertiesPersistingMetadataStore.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2013 the original author or authors. + * Copyright 2002-2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,12 +25,15 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.Properties; +import java.util.concurrent.locks.Lock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.InitializingBean; +import org.springframework.integration.util.DefaultLockRegistry; +import org.springframework.integration.util.LockRegistry; import org.springframework.util.Assert; import org.springframework.util.DefaultPropertiesPersister; @@ -45,7 +48,7 @@ import org.springframework.util.DefaultPropertiesPersister; * @author Gary Russell * @since 2.0 */ -public class PropertiesPersistingMetadataStore implements MetadataStore, InitializingBean, DisposableBean { +public class PropertiesPersistingMetadataStore implements ConcurrentMetadataStore, InitializingBean, DisposableBean { private final Log logger = LogFactory.getLog(getClass()); @@ -53,9 +56,11 @@ public class PropertiesPersistingMetadataStore implements MetadataStore, Initial private final DefaultPropertiesPersister persister = new DefaultPropertiesPersister(); - private volatile File file; + private final LockRegistry lockRegistry = new DefaultLockRegistry(); - private volatile String baseDirectory = System.getProperty("java.io.tmpdir") + "/spring-integration/"; + private String baseDirectory = System.getProperty("java.io.tmpdir") + "/spring-integration/"; + + private File file; public void setBaseDirectory(String baseDirectory) { @@ -82,17 +87,85 @@ public class PropertiesPersistingMetadataStore implements MetadataStore, Initial @Override public void put(String key, String value) { - this.metadata.setProperty(key, value); + Assert.notNull(key, "'key' cannot be null"); + Assert.notNull(value, "'value' cannot be null"); + Lock lock = this.lockRegistry.obtain(key); + lock.lock(); + try { + this.metadata.setProperty(key, value); + } + finally { + lock.unlock(); + } } @Override public String get(String key) { - return this.metadata.getProperty(key); + Assert.notNull(key, "'key' cannot be null"); + Lock lock = this.lockRegistry.obtain(key); + lock.lock(); + try { + return this.metadata.getProperty(key); + } + finally { + lock.unlock(); + } } @Override public String remove(String key) { - return (String) this.metadata.remove(key); + Assert.notNull(key, "'key' cannot be null"); + Lock lock = this.lockRegistry.obtain(key); + lock.lock(); + try { + return (String) this.metadata.remove(key); + } + finally { + lock.unlock(); + } + } + + @Override + public String putIfAbsent(String key, String value) { + Assert.notNull(key, "'key' cannot be null"); + Assert.notNull(value, "'value' cannot be null"); + Lock lock = this.lockRegistry.obtain(key); + lock.lock(); + try { + String property = this.metadata.getProperty(key); + if (property == null) { + this.metadata.setProperty(key, value); + return null; + } + else { + return property; + } + } + finally { + lock.unlock(); + } + } + + @Override + public boolean replace(String key, String oldValue, String newValue) { + Assert.notNull(key, "'key' cannot be null"); + Assert.notNull(oldValue, "'oldValue' cannot be null"); + Assert.notNull(newValue, "'newValue' cannot be null"); + Lock lock = this.lockRegistry.obtain(key); + lock.lock(); + try { + String property = this.metadata.getProperty(key); + if (oldValue.equals(property)) { + this.metadata.setProperty(key, newValue); + return true; + } + else { + return false; + } + } + finally { + lock.unlock(); + } } @Override diff --git a/spring-integration-core/src/main/java/org/springframework/integration/metadata/SimpleMetadataStore.java b/spring-integration-core/src/main/java/org/springframework/integration/metadata/SimpleMetadataStore.java index 0bf5b50fdd..9bf9475feb 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/metadata/SimpleMetadataStore.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/metadata/SimpleMetadataStore.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2013 the original author or authors. + * Copyright 2002-2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -13,8 +13,8 @@ package org.springframework.integration.metadata; -import java.util.HashMap; -import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; /** @@ -22,24 +22,37 @@ import java.util.Map; * The metadata will not be persisted across application restarts. * * @author Mark Fisher + * @author Gary Russell * @since 2.0 */ -public class SimpleMetadataStore implements MetadataStore { +public class SimpleMetadataStore implements ConcurrentMetadataStore { - private final Map metadata = new HashMap(); + private final ConcurrentMap metadata = new ConcurrentHashMap(); + @Override public void put(String key, String value) { this.metadata.put(key, value); } + @Override public String get(String key) { return this.metadata.get(key); } @Override public String remove(String key) { - return metadata.remove(key); + return this.metadata.remove(key); + } + + @Override + public String putIfAbsent(String key, String value) { + return this.metadata.putIfAbsent(key, value); + } + + @Override + public boolean replace(String key, String oldValue, String newValue) { + return this.metadata.replace(key, oldValue, newValue); } } diff --git a/spring-integration-core/src/test/java/org/springframework/integration/metadata/PropertiesPersistingMetadataStoreTests.java b/spring-integration-core/src/test/java/org/springframework/integration/metadata/PropertiesPersistingMetadataStoreTests.java index 8ed8cc371c..9437284663 100644 --- a/spring-integration-core/src/test/java/org/springframework/integration/metadata/PropertiesPersistingMetadataStoreTests.java +++ b/spring-integration-core/src/test/java/org/springframework/integration/metadata/PropertiesPersistingMetadataStoreTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2013 the original author or authors. + * Copyright 2002-2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,9 @@ package org.springframework.integration.metadata; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.io.File; @@ -32,6 +34,7 @@ import org.springframework.core.io.support.PropertiesLoaderUtils; * @author Oleg Zhurakousky * @author Mark Fisher * @author Gunnar Hillert + * @author Gary Russell * @since 2.0 */ public class PropertiesPersistingMetadataStoreTests { @@ -43,7 +46,10 @@ public class PropertiesPersistingMetadataStoreTests { PropertiesPersistingMetadataStore metadataStore = new PropertiesPersistingMetadataStore(); metadataStore.afterPropertiesSet(); assertTrue(file.exists()); - metadataStore.put("foo", "bar"); + assertNull(metadataStore.putIfAbsent("foo", "baz")); + assertNotNull(metadataStore.putIfAbsent("foo", "baz")); + assertFalse(metadataStore.replace("foo", "xxx", "bar")); + assertTrue(metadataStore.replace("foo", "baz", "bar")); metadataStore.destroy(); Properties persistentProperties = PropertiesLoaderUtils.loadProperties(new FileSystemResource(file)); assertNotNull(persistentProperties); diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java index 3a44133ca7..7e0d612594 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 the original author or authors. + * Copyright 2013-2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ package org.springframework.integration.file.filters; -import org.springframework.integration.metadata.MetadataStore; +import org.springframework.integration.metadata.ConcurrentMetadataStore; import org.springframework.util.Assert; /** @@ -30,13 +30,13 @@ import org.springframework.util.Assert; */ public abstract class AbstractPersistentAcceptOnceFileListFilter extends AbstractFileListFilter { - protected final MetadataStore store; + protected final ConcurrentMetadataStore store; protected final String prefix; private final Object monitor = new Object(); - public AbstractPersistentAcceptOnceFileListFilter(MetadataStore store, String prefix) { + public AbstractPersistentAcceptOnceFileListFilter(ConcurrentMetadataStore store, String prefix) { Assert.notNull(store, "'store' cannot be null"); Assert.notNull(prefix, "'prefix' cannot be null"); this.store = store; @@ -47,13 +47,16 @@ public abstract class AbstractPersistentAcceptOnceFileListFilter extends Abst protected boolean accept(F file) { String key = buildKey(file); synchronized(monitor) { - String value = store.get(key); - if (value != null && isEqual(file, value)) { + String newValue = value(file); + String oldValue = this.store.putIfAbsent(key, newValue); + if (oldValue == null) { // not in store + return true; + } + if (isEqual(file, oldValue)) { // same value in store return false; } - store.put(key, value(file)); + return this.store.replace(key, oldValue, newValue); // true if replace successful } - return true; } /** @@ -73,7 +76,7 @@ public abstract class AbstractPersistentAcceptOnceFileListFilter extends Abst * @return true if equal. */ protected boolean isEqual(F file, String value) { - return Long.valueOf(value).longValue() == this.modified(file); + return Long.valueOf(value) == this.modified(file); } /** diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemPersistentAcceptOnceFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemPersistentAcceptOnceFileListFilter.java index 6fb2a2c19f..08f42759fb 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemPersistentAcceptOnceFileListFilter.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemPersistentAcceptOnceFileListFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 the original author or authors. + * Copyright 2013-2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ package org.springframework.integration.file.filters; import java.io.File; -import org.springframework.integration.metadata.MetadataStore; +import org.springframework.integration.metadata.ConcurrentMetadataStore; /** * @author Gary Russell @@ -27,7 +27,7 @@ import org.springframework.integration.metadata.MetadataStore; */ public class FileSystemPersistentAcceptOnceFileListFilter extends AbstractPersistentAcceptOnceFileListFilter { - public FileSystemPersistentAcceptOnceFileListFilter(MetadataStore store, String prefix) { + public FileSystemPersistentAcceptOnceFileListFilter(ConcurrentMetadataStore store, String prefix) { super(store, prefix); } diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterRedisTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterRedisTests.java new file mode 100644 index 0000000000..ce600890c3 --- /dev/null +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterRedisTests.java @@ -0,0 +1,117 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.file.filters; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.serializer.StringRedisSerializer; +import org.springframework.integration.redis.metadata.RedisMetadataStore; +import org.springframework.integration.redis.rules.RedisAvailable; +import org.springframework.integration.redis.rules.RedisAvailableTests; + +/** + * @author Gary Russell + * @since 4.0 + * + */ +public class PersistentAcceptOnceFileListFilterRedisTests extends RedisAvailableTests { + + @Before + @After + public void setupShutDown() { + RedisTemplate template = this.createTemplate(); + template.delete("persistentAcceptOnceFileListFilterRedisTests"); + } + + private RedisTemplate createTemplate() { + RedisTemplate template = new RedisTemplate(); + template.setConnectionFactory(this.getConnectionFactoryForTest()); + template.setKeySerializer(new StringRedisSerializer()); + template.afterPropertiesSet(); + return template; + } + + @Test + @RedisAvailable + public void testFileSystem() throws Exception { + final AtomicBoolean suspend = new AtomicBoolean(); + final CountDownLatch latch1 = new CountDownLatch(1); + final CountDownLatch latch2 = new CountDownLatch(1); + RedisMetadataStore store = new RedisMetadataStore(this.getConnectionFactoryForTest(), + "persistentAcceptOnceFileListFilterRedisTests") { + + @Override + public boolean replace(String key, String oldValue, String newValue) { + if (suspend.get()) { + latch2.countDown(); + try { + latch1.await(10, TimeUnit.SECONDS); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + return super.replace(key, oldValue, newValue); + } + + }; + final FileSystemPersistentAcceptOnceFileListFilter filter = + new FileSystemPersistentAcceptOnceFileListFilter(store,"foo:"); + final File file = File.createTempFile("foo", ".txt"); + assertEquals(1, filter.filterFiles(new File[] {file}).size()); + String ts = store.get("foo:" + file.getAbsolutePath()); + assertEquals(String.valueOf(file.lastModified()), ts); + assertEquals(0, filter.filterFiles(new File[] {file}).size()); + file.setLastModified(file.lastModified() + 5000L); + assertEquals(1, filter.filterFiles(new File[] {file}).size()); + ts = store.get("foo:" + file.getAbsolutePath()); + assertEquals(String.valueOf(file.lastModified()), ts); + assertEquals(0, filter.filterFiles(new File[] {file}).size()); + + suspend.set(true); + file.setLastModified(file.lastModified() + 5000L); + + Future result = Executors.newSingleThreadExecutor().submit(new Callable() { + + @Override + public Integer call() throws Exception { + return filter.filterFiles(new File[] {file}).size(); + } + }); + assertTrue(latch2.await(10, TimeUnit.SECONDS)); + store.put("foo:" + file.getAbsolutePath(), "43"); + latch1.countDown(); + Integer theResult = result.get(10, TimeUnit.SECONDS); + assertEquals(Integer.valueOf(0), theResult); // lost the race, key changed + + file.delete(); + } + +} diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterTests.java index 0fe1e51dbd..fe29d12e53 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 the original author or authors. + * Copyright 2013-2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,20 @@ */ package org.springframework.integration.file.filters; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.File; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.Test; -import org.springframework.integration.metadata.MetadataStore; +import org.springframework.integration.metadata.ConcurrentMetadataStore; import org.springframework.integration.metadata.SimpleMetadataStore; /** @@ -33,14 +40,55 @@ public class PersistentAcceptOnceFileListFilterTests { @Test public void testFileSystem() throws Exception { - MetadataStore store = new SimpleMetadataStore(); - FileSystemPersistentAcceptOnceFileListFilter filter = new FileSystemPersistentAcceptOnceFileListFilter(store, "foo:"); - File file = File.createTempFile("foo", ".txt"); - assertTrue(filter.filterFiles(new File[] {file}).size() == 1); - assertTrue(filter.filterFiles(new File[] {file}).size() == 0); - file.setLastModified(27L); - assertTrue(filter.filterFiles(new File[] {file}).size() == 1); - assertTrue(filter.filterFiles(new File[] {file}).size() == 0); + final AtomicBoolean suspend = new AtomicBoolean(); + final CountDownLatch latch1 = new CountDownLatch(1); + final CountDownLatch latch2 = new CountDownLatch(1); + ConcurrentMetadataStore store = new SimpleMetadataStore() { + + @Override + public boolean replace(String key, String oldValue, String newValue) { + if (suspend.get()) { + latch2.countDown(); + try { + latch1.await(10, TimeUnit.SECONDS); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + return super.replace(key, oldValue, newValue); + } + + }; + final FileSystemPersistentAcceptOnceFileListFilter filter = + new FileSystemPersistentAcceptOnceFileListFilter(store, "foo:"); + final File file = File.createTempFile("foo", ".txt"); + assertEquals(1, filter.filterFiles(new File[] {file}).size()); + String ts = store.get("foo:" + file.getAbsolutePath()); + assertEquals(String.valueOf(file.lastModified()), ts); + assertEquals(0, filter.filterFiles(new File[] {file}).size()); + file.setLastModified(file.lastModified() + 5000L); + assertEquals(1, filter.filterFiles(new File[] {file}).size()); + ts = store.get("foo:" + file.getAbsolutePath()); + assertEquals(String.valueOf(file.lastModified()), ts); + assertEquals(0, filter.filterFiles(new File[] {file}).size()); + + suspend.set(true); + file.setLastModified(file.lastModified() + 5000L); + + Future result = Executors.newSingleThreadExecutor().submit(new Callable() { + + @Override + public Integer call() throws Exception { + return filter.filterFiles(new File[] {file}).size(); + } + }); + assertTrue(latch2.await(10, TimeUnit.SECONDS)); + store.put("foo:" + file.getAbsolutePath(), "43"); + latch1.countDown(); + Integer theResult = result.get(10, TimeUnit.SECONDS); + assertEquals(Integer.valueOf(0), theResult); // lost the race, key changed + file.delete(); } diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpPersistentAcceptOnceFileListFilter.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpPersistentAcceptOnceFileListFilter.java index 91ae1bd3f3..29971991cb 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpPersistentAcceptOnceFileListFilter.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpPersistentAcceptOnceFileListFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 the original author or authors. + * Copyright 2013-2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,12 +19,11 @@ package org.springframework.integration.ftp.filters; import org.apache.commons.net.ftp.FTPFile; import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter; -import org.springframework.integration.metadata.MetadataStore; +import org.springframework.integration.metadata.ConcurrentMetadataStore; /** - * Since the super class deems files as 'not seen' if the timestamp is different, remote file - * users should use the adapter's preserve-timestamp option. Otherwise if a file is re-fetched - * it will have a new timestamp. + * Persistent file list filter using the server's file timestamp to detect if we've already + * 'seen' this file. * * @author Gary Russell * @since 3.0 @@ -32,7 +31,7 @@ import org.springframework.integration.metadata.MetadataStore; */ public class FtpPersistentAcceptOnceFileListFilter extends AbstractPersistentAcceptOnceFileListFilter { - public FtpPersistentAcceptOnceFileListFilter(MetadataStore store, String prefix) { + public FtpPersistentAcceptOnceFileListFilter(ConcurrentMetadataStore store, String prefix) { super(store, prefix); } diff --git a/spring-integration-redis/src/main/java/org/springframework/integration/redis/metadata/RedisMetadataStore.java b/spring-integration-redis/src/main/java/org/springframework/integration/redis/metadata/RedisMetadataStore.java index 9506a62a96..76590df685 100644 --- a/spring-integration-redis/src/main/java/org/springframework/integration/redis/metadata/RedisMetadataStore.java +++ b/spring-integration-redis/src/main/java/org/springframework/integration/redis/metadata/RedisMetadataStore.java @@ -18,6 +18,7 @@ import org.springframework.data.redis.core.BoundHashOperations; import org.springframework.data.redis.core.RedisOperations; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.data.redis.support.collections.RedisProperties; +import org.springframework.integration.metadata.ConcurrentMetadataStore; import org.springframework.integration.metadata.MetadataStore; import org.springframework.util.Assert; @@ -29,7 +30,7 @@ import org.springframework.util.Assert; * @author Artem Bilan * @since 3.0 */ -public class RedisMetadataStore implements MetadataStore { +public class RedisMetadataStore implements ConcurrentMetadataStore { public static final String KEY = "MetaData"; @@ -113,14 +114,41 @@ public class RedisMetadataStore implements MetadataStore { @Override public String get(String key) { Assert.notNull(key, "'key' must not be null."); - return (String) this.properties.get(key); + Object value = this.properties.get(key); + if (value != null) { + Assert.isInstanceOf(String.class, value, "Invalid type in the store"); + } + return (String) value; } @Override public String remove(String key) { Assert.notNull(key, "'key' must not be null."); - return (String) this.properties.remove(key); + Object removed = this.properties.remove(key); + if (removed != null) { + Assert.isInstanceOf(String.class, removed, "The removed value was an invalid type"); + } + return (String) removed; + } + + @Override + public String putIfAbsent(String key, String value) { + Assert.notNull(key, "'key' must not be null."); + Assert.notNull(value, "'value' must not be null."); + Object oldValue = this.properties.putIfAbsent(key, value); + if (oldValue != null) { + Assert.isInstanceOf(String.class, oldValue, "Invalid type in the store"); + } + return (String) oldValue; + } + + @Override + public boolean replace(String key, String oldValue, String newValue) { + Assert.notNull(key, "'key' must not be null."); + Assert.notNull(oldValue, "'oldValue' must not be null."); + Assert.notNull(newValue, "'newValue' must not be null."); + return this.properties.replace(key, oldValue, newValue); } } diff --git a/spring-integration-redis/src/test/java/org/springframework/integration/redis/util/RedisLockRegistryTests.java b/spring-integration-redis/src/test/java/org/springframework/integration/redis/util/RedisLockRegistryTests.java index 6868accd87..1f501ecf30 100644 --- a/spring-integration-redis/src/test/java/org/springframework/integration/redis/util/RedisLockRegistryTests.java +++ b/spring-integration-redis/src/test/java/org/springframework/integration/redis/util/RedisLockRegistryTests.java @@ -57,7 +57,7 @@ public class RedisLockRegistryTests extends RedisAvailableTests { @Before @After - public void shutDown() { + public void setupShutDown() { RedisTemplate template = this.createTemplate(); template.delete("rlrTests"); template.delete("rlrTests2"); diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpPersistentAcceptOnceFileListFilter.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpPersistentAcceptOnceFileListFilter.java index 0140c2d8e2..6dc09c305b 100644 --- a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpPersistentAcceptOnceFileListFilter.java +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpPersistentAcceptOnceFileListFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 the original author or authors. + * Copyright 2013-2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,14 +18,13 @@ package org.springframework.integration.sftp.filters; import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter; -import org.springframework.integration.metadata.MetadataStore; +import org.springframework.integration.metadata.ConcurrentMetadataStore; import com.jcraft.jsch.ChannelSftp.LsEntry; /** - * Since the super class deems files as 'not seen' if the timestamp is different, remote file - * users should use the adapter's preserve-timestamp option. Otherwise if a file is re-fetched - * it will have a new timestamp. + * Persistent file list filter using the server's file timestamp to detect if we've already + * 'seen' this file. * * @author Gary Russell * @since 3.0 @@ -33,7 +32,7 @@ import com.jcraft.jsch.ChannelSftp.LsEntry; */ public class SftpPersistentAcceptOnceFileListFilter extends AbstractPersistentAcceptOnceFileListFilter { - public SftpPersistentAcceptOnceFileListFilter(MetadataStore store, String prefix) { + public SftpPersistentAcceptOnceFileListFilter(ConcurrentMetadataStore store, String prefix) { super(store, prefix); } diff --git a/src/reference/docbook/aggregator.xml b/src/reference/docbook/aggregator.xml index 451d824fde..24d6ee7eda 100644 --- a/src/reference/docbook/aggregator.xml +++ b/src/reference/docbook/aggregator.xml @@ -535,6 +535,9 @@ then you should simply provide an implementation of the ReleaseStrate used to obtain a Lock based on the groupId for concurrent operations on the MessageGroup. By default, an internal DefaultLockRegistry is used. + Use of a distributed LockRegistry, such as the RedisLockRegistry, ensures only one instance of the aggregator will operate + on a group concurrently. See for more information. diff --git a/src/reference/docbook/file.xml b/src/reference/docbook/file.xml index 99cfd6116b..1ce101d70e 100644 --- a/src/reference/docbook/file.xml +++ b/src/reference/docbook/file.xml @@ -36,11 +36,21 @@ AcceptOnceFileListFilter is used. This filter ensures files are picked up only once from the directory. + The AcceptOnceFileListFilter stores its state in memory. If you wish the state to survive a system restart, consider using the FileSystemPersistentAcceptOnceFileListFilter instead. This filter stores - the accepted file names in a MetadataStore strategy (). + the accepted file names in a MetadataStore implementation + (). This filter matches on the filename and modified time. + + + Since version 4.0, this filter requires a ConcurrentMetadataStore. When used with a shared data store + (such as Redis with the RedisMetadataStore) this allows + filter keys to be shared across multiple application instances, or when a network + file share is being used by multiple servers. + MetadataStore strategy (). This filter matches on the filename and the remote modified time. + + + Since version 4.0, this filter requires a ConcurrentMetadataStore. When used with a shared data store + (such as Redis with the RedisMetadataStore) this allows + filter keys to be shared across multiple application or server instances. diff --git a/src/reference/docbook/meta-data-store.xml b/src/reference/docbook/meta-data-store.xml index f242f3c0da..54eb63fab5 100644 --- a/src/reference/docbook/meta-data-store.xml +++ b/src/reference/docbook/meta-data-store.xml @@ -40,6 +40,12 @@ MetadataStore interface (e.g. JdbcMetadataStore) and configure it as a bean in the Application Context. + + Starting with version 4.0, SimpleMetadataStore, + PropertiesPersistingMetadataStore and + RedisMetadataStore implement ConcurrentMetadataStore. + These provide for atomic updates and can be used across multiple component or application instances. +
Idempotent Receiver diff --git a/src/reference/docbook/redis.xml b/src/reference/docbook/redis.xml index 2ecf9ae2ed..af7b398e3b 100644 --- a/src/reference/docbook/redis.xml +++ b/src/reference/docbook/redis.xml @@ -489,6 +489,11 @@ rt.setConnectionFactory(redisConnectionFactory);]]> key plays the role of a region, which is useful in distributed environment, when several applications use the same Redis server. By default this key has the value MetaData. + + Starting with version 4.0, this store now implements ConcurrentMetadataStore, allowing it to be reliably shared across multiple + application instances where only one instance will be allowed to store or modify a key's value. +
RedisStore Inbound Channel Adapter @@ -770,4 +775,34 @@ the serialization of values, you may want to consider providing your own Redis Specification.
+
+ Redis Lock Registry + + Starting with version 4.0, the RedisLockRegistry is + available. Certain components (for example aggregator and resequencer) use a lock obtained from + a LockRegistry instance to ensure + that only one thread is manipulating a group at a time. The DefaultLockRegistry + performs this function within a single component; you can now configure an external lock registry + on these components. When used with a shared MessageGroupStore, + the RedisLockRegistry can be use to provide this functionality across + multiple application instances, such that only one instance can manipulate the group at a time. + + + When a lock is released by a local thread, another local thread will generally be able to acquire the + lock immediately. If a lock is released by a thread using a different registry instance, it can take up to + 100ms to acquire the lock. + + + To avoid "hung" locks (when a server fails), the locks in this registry are expired after a default + 60 seconds, but this can be configured on the registry. Locks are normally held for a much smaller + time. + + + Because the keys can expire, an attempt to unlock an expired lock will result in an exception + being thrown. However, be aware that the resources protected by such a lock may have been + compromised so such exceptions should be considered severe. The expiry should be set at + a large enough value to prevent this condition, while small enough that the lock can + be recovered after a server failure in a reasonable amount of time. + +
diff --git a/src/reference/docbook/resequencer.xml b/src/reference/docbook/resequencer.xml index 7c249b04af..1dada919d1 100644 --- a/src/reference/docbook/resequencer.xml +++ b/src/reference/docbook/resequencer.xml @@ -56,9 +56,9 @@ lock-registry="lockRegistry" ]]> ]]>
+ group-timeout="60000" ]]> ]]> The id of the resequencer is diff --git a/src/reference/docbook/sftp.xml b/src/reference/docbook/sftp.xml index 63fc8aa1d5..4630533847 100644 --- a/src/reference/docbook/sftp.xml +++ b/src/reference/docbook/sftp.xml @@ -320,6 +320,12 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp the accepted file names in an instance of the MetadataStore strategy (). This filter matches on the filename and the remote modified time. + + + Since version 4.0, this filter requires a ConcurrentMetadataStore. When used with a shared data store + (such as Redis with the RedisMetadataStore) this allows + filter keys to be shared across multiple application or server instances. diff --git a/src/reference/docbook/whats-new.xml b/src/reference/docbook/whats-new.xml index 8901015c8e..7cf08d0c37 100644 --- a/src/reference/docbook/whats-new.xml +++ b/src/reference/docbook/whats-new.xml @@ -133,6 +133,15 @@ For more information, see . +
+ RedisLockRegistry + + The RedisLockRegistry is now available supporting global locks visible + to multiple application instances/servers. This can be used with aggregating message handlers + across multiple application instances such that group release will occur on only one instance. + For more information, see and . + +
@@ -220,5 +229,17 @@ For more information see .
+
+ Redis Metadata Store + + The RedisMetadataStore now implements ConcurrentMetadataStore, allowing it to be used, for example, + in a AbstractPersistentAcceptOnceFileListFilter + implementation in a multiple application instance/server environment. + For more information, see , + , and + . + +