Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-3572: Improve Accept Once Filter Performance

Browse Source 
JIRA: https://jira.spring.io/browse/INT-3572

Previously, a blocking queue was used to hold seen files; this was used to enable
FIFO when a max capacity is set.

When used with no capacity, a queue is not needed; also the `contains` operation
on a queue requires a linear search which does not scale well for a large number
of files.

Use a `HashSet` instead to significantly improve the `contains` performance.

Only maintain a queue if a max capacity is set.

Fix `AcceptOnceFileListFilterTests` to be compatible with Java < 8
This commit is contained in:
Gary Russell
2014-12-08 16:37:07 -05:00
committed by Artem Bilan
parent 1f23bf8f0b
commit e714a5a2a8
2 changed files with 64 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
spring-integration-file/src/main/java/org/springframework/integration/file/filters/AcceptOnceFileListFilter.java
View File

@@ -16,8 +16,10 @@
package org.springframework.integration.file.filters;
import java.util.HashSet;
import java.util.List;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;
/**
@@ -36,6 +38,8 @@ public class AcceptOnceFileListFilter<F> extends AbstractFileListFilter<F> imple
private final Queue<F> seen;
private final Set<F> seenSet = new HashSet<F>();
private final Object monitor = new Object();
@@ -54,20 +58,24 @@ public class AcceptOnceFileListFilter<F> extends AbstractFileListFilter<F> imple
* Creates an AcceptOnceFileListFilter based on an unbounded queue.
*/
public AcceptOnceFileListFilter() {
this.seen = new LinkedBlockingQueue<F>();
this.seen = null;
}
@Override
public boolean accept(F file) {
synchronized (this.monitor) {
if (this.seen.contains(file)) {
if (this.seenSet.contains(file)) {
return false;
}
if (!this.seen.offer(file)) {
this.seen.poll();
this.seen.add(file);
if (this.seen != null) {
if (!this.seen.offer(file)) {
F removed = this.seen.poll();
this.seenSet.remove(removed);
this.seen.add(file);
}
}
this.seenSet.add(file);
return true;
}
}
@@ -78,13 +86,18 @@ public class AcceptOnceFileListFilter<F> extends AbstractFileListFilter<F> imple
*/
@Override
public void rollback(F file, List<F> files) {
boolean rollingBack = false;
for (F fileToRollback : files) {
if (fileToRollback.equals(file)) {
rollingBack = true;
}
if (rollingBack) {
this.seen.remove(fileToRollback);
synchronized (this.monitor) {
boolean rollingBack = false;
for (F fileToRollback : files) {
if (fileToRollback.equals(file)) {
rollingBack = true;
}
if (rollingBack) {
this.seenSet.remove(fileToRollback);
if (this.seen != null) {
this.seen.remove(fileToRollback);
}
}
}
}
}

39
spring-integration-file/src/test/java/org/springframework/integration/file/filters/AcceptOnceFileListFilterTests.java
View File

@@ -16,14 +16,23 @@
package org.springframework.integration.file.filters;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import java.util.Arrays;
import java.util.List;
import java.util.Queue;
import java.util.Set;
import org.junit.Test;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.util.StopWatch;
/**
* @author Gary Russell
* @since 4.0.4
@@ -31,6 +40,36 @@ import org.junit.Test;
*/
public class AcceptOnceFileListFilterTests {
@Test
// This test used to take 34 seconds to run; now 25 milliseconds.
public void testPerformance_INT3572() {
StopWatch watch = new StopWatch();
watch.start();
AcceptOnceFileListFilter<String> filter = new AcceptOnceFileListFilter<String>();
for (int i = 0; i < 100000; i++) {
filter.accept("" + i);
}
watch.stop();
assertTrue(watch.getTotalTimeMillis() < 5000);
}
@Test
@SuppressWarnings("unchecked")
public void testCapacity() {
AcceptOnceFileListFilter<String> filter = new AcceptOnceFileListFilter<String>(2);
assertTrue(filter.accept("foo"));
assertTrue(filter.accept("bar"));
assertFalse(filter.accept("foo"));
assertTrue(filter.accept("baz"));
assertTrue(filter.accept("foo"));
Queue<String> seen = TestUtils.getPropertyValue(filter, "seen", Queue.class);
assertEquals(2, seen.size());
Set<String> seenSet = TestUtils.getPropertyValue(filter, "seenSet", Set.class);
assertEquals(2, seenSet.size());
assertThat(seen, contains("baz", "foo"));
assertThat(seenSet, containsInAnyOrder("foo", "baz"));
}
@Test
public void testRollback() {
AcceptOnceFileListFilter<String> filter = new AcceptOnceFileListFilter<String>();