Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-3353 Concurrent RedisMetadataStore

Browse Source 
JIRA: https://jira.spring.io/browse/INT-3353

Shared metadata for PersistentAcceptOnceFileListFilters.

INT-3353 Polishing; PR Comments
This commit is contained in:
Gary Russell
2014-04-07 17:37:55 +03:00
committed by Artem Bilan
parent 40a535b140
commit 4dee2a224b
21 changed files with 482 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2013 the original author or authors.
* Copyright 2013-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
*/
package org.springframework.integration.file.filters;
import org.springframework.integration.metadata.MetadataStore;
import org.springframework.integration.metadata.ConcurrentMetadataStore;
import org.springframework.util.Assert;
/**
@@ -30,13 +30,13 @@ import org.springframework.util.Assert;
*/
public abstract class AbstractPersistentAcceptOnceFileListFilter<F> extends AbstractFileListFilter<F> {
protected final MetadataStore store;
protected final ConcurrentMetadataStore store;
protected final String prefix;
private final Object monitor = new Object();
public AbstractPersistentAcceptOnceFileListFilter(MetadataStore store, String prefix) {
public AbstractPersistentAcceptOnceFileListFilter(ConcurrentMetadataStore store, String prefix) {
Assert.notNull(store, "'store' cannot be null");
Assert.notNull(prefix, "'prefix' cannot be null");
this.store = store;
@@ -47,13 +47,16 @@ public abstract class AbstractPersistentAcceptOnceFileListFilter<F> extends Abst
protected boolean accept(F file) {
String key = buildKey(file);
synchronized(monitor) {
String value = store.get(key);
if (value != null && isEqual(file, value)) {
String newValue = value(file);
String oldValue = this.store.putIfAbsent(key, newValue);
if (oldValue == null) { // not in store
return true;
}
if (isEqual(file, oldValue)) { // same value in store
return false;
}
store.put(key, value(file));
return this.store.replace(key, oldValue, newValue); // true if replace successful
}
return true;
}
/**
@@ -73,7 +76,7 @@ public abstract class AbstractPersistentAcceptOnceFileListFilter<F> extends Abst
* @return true if equal.
*/
protected boolean isEqual(F file, String value) {
return Long.valueOf(value).longValue() == this.modified(file);
return Long.valueOf(value) == this.modified(file);
}
/**

6
spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemPersistentAcceptOnceFileListFilter.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2013 the original author or authors.
* Copyright 2013-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@ package org.springframework.integration.file.filters;
import java.io.File;
import org.springframework.integration.metadata.MetadataStore;
import org.springframework.integration.metadata.ConcurrentMetadataStore;
/**
* @author Gary Russell
@@ -27,7 +27,7 @@ import org.springframework.integration.metadata.MetadataStore;
*/
public class FileSystemPersistentAcceptOnceFileListFilter extends AbstractPersistentAcceptOnceFileListFilter<File> {
public FileSystemPersistentAcceptOnceFileListFilter(MetadataStore store, String prefix) {
public FileSystemPersistentAcceptOnceFileListFilter(ConcurrentMetadataStore store, String prefix) {
super(store, prefix);
}

117
spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterRedisTests.java Normal file
View File

@@ -0,0 +1,117 @@
/*
* Copyright 2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.file.filters;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.StringRedisSerializer;
import org.springframework.integration.redis.metadata.RedisMetadataStore;
import org.springframework.integration.redis.rules.RedisAvailable;
import org.springframework.integration.redis.rules.RedisAvailableTests;
/**
* @author Gary Russell
* @since 4.0
*
*/
public class PersistentAcceptOnceFileListFilterRedisTests extends RedisAvailableTests {
@Before
@After
public void setupShutDown() {
RedisTemplate<String, ?> template = this.createTemplate();
template.delete("persistentAcceptOnceFileListFilterRedisTests");
}
private RedisTemplate<String, ?> createTemplate() {
RedisTemplate<String, ?> template = new RedisTemplate<String, Object>();
template.setConnectionFactory(this.getConnectionFactoryForTest());
template.setKeySerializer(new StringRedisSerializer());
template.afterPropertiesSet();
return template;
}
@Test
@RedisAvailable
public void testFileSystem() throws Exception {
final AtomicBoolean suspend = new AtomicBoolean();
final CountDownLatch latch1 = new CountDownLatch(1);
final CountDownLatch latch2 = new CountDownLatch(1);
RedisMetadataStore store = new RedisMetadataStore(this.getConnectionFactoryForTest(),
"persistentAcceptOnceFileListFilterRedisTests") {
@Override
public boolean replace(String key, String oldValue, String newValue) {
if (suspend.get()) {
latch2.countDown();
try {
latch1.await(10, TimeUnit.SECONDS);
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
return super.replace(key, oldValue, newValue);
}
};
final FileSystemPersistentAcceptOnceFileListFilter filter =
new FileSystemPersistentAcceptOnceFileListFilter(store,"foo:");
final File file = File.createTempFile("foo", ".txt");
assertEquals(1, filter.filterFiles(new File[] {file}).size());
String ts = store.get("foo:" + file.getAbsolutePath());
assertEquals(String.valueOf(file.lastModified()), ts);
assertEquals(0, filter.filterFiles(new File[] {file}).size());
file.setLastModified(file.lastModified() + 5000L);
assertEquals(1, filter.filterFiles(new File[] {file}).size());
ts = store.get("foo:" + file.getAbsolutePath());
assertEquals(String.valueOf(file.lastModified()), ts);
assertEquals(0, filter.filterFiles(new File[] {file}).size());
suspend.set(true);
file.setLastModified(file.lastModified() + 5000L);
Future<Integer> result = Executors.newSingleThreadExecutor().submit(new Callable<Integer>() {
@Override
public Integer call() throws Exception {
return filter.filterFiles(new File[] {file}).size();
}
});
assertTrue(latch2.await(10, TimeUnit.SECONDS));
store.put("foo:" + file.getAbsolutePath(), "43");
latch1.countDown();
Integer theResult = result.get(10, TimeUnit.SECONDS);
assertEquals(Integer.valueOf(0), theResult); // lost the race, key changed
file.delete();
}
}

68
spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2013 the original author or authors.
* Copyright 2013-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,13 +15,20 @@
*/
package org.springframework.integration.file.filters;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Test;
import org.springframework.integration.metadata.MetadataStore;
import org.springframework.integration.metadata.ConcurrentMetadataStore;
import org.springframework.integration.metadata.SimpleMetadataStore;
/**
@@ -33,14 +40,55 @@ public class PersistentAcceptOnceFileListFilterTests {
@Test
public void testFileSystem() throws Exception {
MetadataStore store = new SimpleMetadataStore();
FileSystemPersistentAcceptOnceFileListFilter filter = new FileSystemPersistentAcceptOnceFileListFilter(store, "foo:");
File file = File.createTempFile("foo", ".txt");
assertTrue(filter.filterFiles(new File[] {file}).size() == 1);
assertTrue(filter.filterFiles(new File[] {file}).size() == 0);
file.setLastModified(27L);
assertTrue(filter.filterFiles(new File[] {file}).size() == 1);
assertTrue(filter.filterFiles(new File[] {file}).size() == 0);
final AtomicBoolean suspend = new AtomicBoolean();
final CountDownLatch latch1 = new CountDownLatch(1);
final CountDownLatch latch2 = new CountDownLatch(1);
ConcurrentMetadataStore store = new SimpleMetadataStore() {
@Override
public boolean replace(String key, String oldValue, String newValue) {
if (suspend.get()) {
latch2.countDown();
try {
latch1.await(10, TimeUnit.SECONDS);
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
return super.replace(key, oldValue, newValue);
}
};
final FileSystemPersistentAcceptOnceFileListFilter filter =
new FileSystemPersistentAcceptOnceFileListFilter(store, "foo:");
final File file = File.createTempFile("foo", ".txt");
assertEquals(1, filter.filterFiles(new File[] {file}).size());
String ts = store.get("foo:" + file.getAbsolutePath());
assertEquals(String.valueOf(file.lastModified()), ts);
assertEquals(0, filter.filterFiles(new File[] {file}).size());
file.setLastModified(file.lastModified() + 5000L);
assertEquals(1, filter.filterFiles(new File[] {file}).size());
ts = store.get("foo:" + file.getAbsolutePath());
assertEquals(String.valueOf(file.lastModified()), ts);
assertEquals(0, filter.filterFiles(new File[] {file}).size());
suspend.set(true);
file.setLastModified(file.lastModified() + 5000L);
Future<Integer> result = Executors.newSingleThreadExecutor().submit(new Callable<Integer>() {
@Override
public Integer call() throws Exception {
return filter.filterFiles(new File[] {file}).size();
}
});
assertTrue(latch2.await(10, TimeUnit.SECONDS));
store.put("foo:" + file.getAbsolutePath(), "43");
latch1.countDown();
Integer theResult = result.get(10, TimeUnit.SECONDS);
assertEquals(Integer.valueOf(0), theResult); // lost the race, key changed
file.delete();
}