Incomplete: Improve FileSource

- refactored Backlog to be more robust (still missing concurrent testcases)
   - refactored FileInfo into FileSnapshot (to remove need for maps in backlog)
   - refactored testcases and AbstractDirectorySource accordingly
This commit is contained in:
Iwein Fuld
2008-08-24 13:24:15 +00:00
parent eba716e65b
commit 8559f2f74b
10 changed files with 558 additions and 415 deletions

View File

@@ -0,0 +1,167 @@
/*
* Copyright 2002-2008 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.adapter.file;
import java.util.ArrayList;
import java.util.concurrent.PriorityBlockingQueue;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.DirectFieldAccessor;
import org.springframework.integration.adapter.file.Backlog;
import org.springframework.integration.adapter.file.FileSnapshot;
/**
* @author Marius Bogoevici
* @author Iwein Fuld
*/
@SuppressWarnings("unchecked")
public class BacklogTests {
private Backlog<FileSnapshot> backlog;
private ArrayList<FileSnapshot> remoteSnapshot;
private FileSnapshot[] process;
@Before
public void setUp() {
backlog = new Backlog<FileSnapshot>();
process = new FileSnapshot[5];
}
@Test
public void testInitialization() {
Assert.assertTrue(backlog.isEmpty());
backlog.processSnapshot(remoteSnapshot);
PriorityBlockingQueue<FileSnapshot> queue = (PriorityBlockingQueue<FileSnapshot>) new DirectFieldAccessor(
backlog).getPropertyValue("backlog");
Assert.assertEquals(3, queue.size());
Assert.assertTrue(queue.containsAll(remoteSnapshot));
}
@Test
public void testFullProcessingInOneStep() {
backlog.processSnapshot(remoteSnapshot);
backlog.fileProcessed( remoteSnapshot.toArray(process));
Assert.assertTrue(backlog.isEmpty());
backlog.processSnapshot(remoteSnapshot);
Assert.assertTrue(backlog.isEmpty());
}
@Test
public void testFullProcessingInTwoSteps() {
backlog.processSnapshot(remoteSnapshot);
backlog.fileProcessed( remoteSnapshot.subList(0, 2).toArray(process));
PriorityBlockingQueue<FileSnapshot> queue = (PriorityBlockingQueue<FileSnapshot>) new DirectFieldAccessor(
backlog).getPropertyValue("backlog");
Assert.assertEquals(1, queue.size());
Assert.assertTrue(queue.contains(remoteSnapshot.get(2)));
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(1, queue.size());
Assert.assertTrue(queue.contains(remoteSnapshot.get(2)));
backlog.fileProcessed(remoteSnapshot.get(2));
Assert.assertTrue(backlog.isEmpty());
backlog.processSnapshot(remoteSnapshot);
Assert.assertTrue(backlog.isEmpty());
}
@Test
public void testOneFileChangedSize() {
PriorityBlockingQueue<FileSnapshot> queue = (PriorityBlockingQueue<FileSnapshot>) new DirectFieldAccessor(
backlog).getPropertyValue("backlog");
backlog.processSnapshot(remoteSnapshot);
backlog.fileProcessed( remoteSnapshot.toArray(process));
Assert.assertTrue(backlog.isEmpty());
backlog.processSnapshot(remoteSnapshot);
Assert.assertTrue(backlog.isEmpty());
remoteSnapshot.remove(2);
FileSnapshot modifiedC = new FileSnapshot("c.txt", 1001, 112);
remoteSnapshot.add(modifiedC);
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(1, queue.size());
Assert.assertTrue(queue.contains(modifiedC));
}
@Test
public void testOneFileChangedDate() {
PriorityBlockingQueue<FileSnapshot> queue = (PriorityBlockingQueue<FileSnapshot>) new DirectFieldAccessor(
backlog).getPropertyValue("backlog");
backlog.processSnapshot(remoteSnapshot);
backlog.fileProcessed( remoteSnapshot.toArray(process));
Assert.assertTrue(backlog.isEmpty());
backlog.processSnapshot(remoteSnapshot);
remoteSnapshot.remove(2);
FileSnapshot modifiedC = new FileSnapshot("c.txt", 1011, 102);
remoteSnapshot.add(modifiedC);
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(1, queue.size());
Assert.assertTrue(queue.contains(modifiedC));
}
@Test
public void testOneFileAdded() {
PriorityBlockingQueue<FileSnapshot> queue = (PriorityBlockingQueue<FileSnapshot>) new DirectFieldAccessor(
backlog).getPropertyValue("backlog");
backlog.processSnapshot(remoteSnapshot);
backlog.fileProcessed( remoteSnapshot.toArray(process));
Assert.assertTrue(backlog.isEmpty());
backlog.processSnapshot(remoteSnapshot);
FileSnapshot newD = new FileSnapshot("d.txt", 1003, 103);
remoteSnapshot.add(newD);
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(1, queue.size());
Assert.assertTrue(queue.contains(newD));
}
@Test
public void testOneFileRemoved() {
backlog.processSnapshot(remoteSnapshot);
backlog.fileProcessed( remoteSnapshot.toArray(process));
Assert.assertTrue(backlog.isEmpty());
backlog.processSnapshot(remoteSnapshot);
remoteSnapshot.remove(2);
backlog.processSnapshot(remoteSnapshot);
Assert.assertTrue(backlog.isEmpty());
}
@Test
public void testOneFileRemovedBeforeBeingProcessedInTheNextStep() {
PriorityBlockingQueue<FileSnapshot> queue = (PriorityBlockingQueue<FileSnapshot>) new DirectFieldAccessor(
backlog).getPropertyValue("backlog");
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(3, queue.size());
remoteSnapshot.remove(2);
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(2, queue.size());
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(2, queue.size());
}
// @Test selectForProcessing success/failure
@Before
public void generateInitialSnapshot() {
this.remoteSnapshot = new ArrayList<FileSnapshot>();
remoteSnapshot.add(new FileSnapshot("a.txt", 1000, 100));
remoteSnapshot.add(new FileSnapshot("b.txt", 1001, 101));
remoteSnapshot.add(new FileSnapshot("c.txt", 1002, 102));
}
}

View File

@@ -0,0 +1,103 @@
/*
* Copyright 2002-2008 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.adapter.file;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.springframework.core.io.FileSystemResource;
import org.springframework.integration.message.Message;
/**
* @author Iwein Fuld
*/
public class FileSourceTests {
private FileSource fileSource;
private static final String INPUT_DIR = System.getProperty("java.io.tmpdir") + "/"
+ FileSourceTests.class.getCanonicalName();
@BeforeClass
public static void createTmpDir() {
new File(INPUT_DIR).mkdirs();
}
@Before
public void refreshFileSource() {
fileSource = new FileSource(new FileSystemResource(INPUT_DIR));
}
@After
public void cleanOutTmp() {
for (File file : new File(INPUT_DIR).listFiles()) {
file.delete();
}
}
@AfterClass
public static void removeTmp() {
if (!new File(INPUT_DIR).delete()) {
throw new RuntimeException("failed to clean up directory [" + INPUT_DIR + "]");
}
}
@Test
public void testNoFile() {
assertNull("There should be no message on the source", fileSource.receive());
}
@Test
public void closedEmptyFile() throws Exception {
new File(INPUT_DIR + "/test").createNewFile();
assertNotNull("No file received after writing to input directory", fileSource.receive());
}
@Test
public void closedWriter() throws Exception {
BufferedWriter writer = new BufferedWriter(new FileWriter(new File(INPUT_DIR + "/test")));
writer.write("some stuff");
writer.close();
assertNotNull("No file received after writing to input directory", fileSource.receive());
}
@Test
/*
* This test shows the how not to do it (i.e. without a proper external
* trigger)
*/
public void testOpenWriter() throws Exception {
BufferedWriter writer = new BufferedWriter(new FileWriter(new File(INPUT_DIR + "/test")));
writer.write("some stuff");
// don't close the writer yet
Message<File> received = fileSource.receive();
assertNotNull("incomplete file was not received", received);
fileSource.onSend(received);
writer.write("some more stuff");
writer.close();
assertNotNull("something shoulda happened don't you think?", received);
}
}

View File

@@ -1,180 +0,0 @@
/*
* Copyright 2002-2008 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.adapter.ftp;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.integration.adapter.file.Backlog;
import org.springframework.integration.adapter.file.FileInfo;
/**
* @author Marius Bogoevici
* @author Iwein Fuld
*/
public class BacklogTests {
private Backlog<FileInfo> backlog;
@Before
public void setUp() {
backlog = new Backlog<FileInfo>();
}
@Test
public void testInitialization() {
Assert.assertTrue(backlog.getBacklog().isEmpty());
Map<String, FileInfo> remoteSnapshot = generateInitialSnapshot();
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(3, backlog.getBacklog().size());
Assert.assertTrue(backlog.getBacklog().containsKey("a.txt"));
Assert.assertTrue(backlog.getBacklog().containsKey("b.txt"));
Assert.assertTrue(backlog.getBacklog().containsKey("c.txt"));
}
@Test
public void testFullProcessingInOneStep() {
Assert.assertTrue(backlog.getBacklog().isEmpty());
Map<String, FileInfo> remoteSnapshot = generateInitialSnapshot();
backlog.processSnapshot(remoteSnapshot);
backlog.fileProcessed("a.txt");
backlog.fileProcessed("b.txt");
backlog.fileProcessed("c.txt");
Assert.assertTrue(backlog.getBacklog().isEmpty());
backlog.processSnapshot(remoteSnapshot);
Assert.assertTrue(backlog.getBacklog().isEmpty());
}
@Test
public void testFullProcessingInTwoSteps() {
Assert.assertTrue(backlog.getBacklog().isEmpty());
Map<String, FileInfo> remoteSnapshot = generateInitialSnapshot();
backlog.processSnapshot(remoteSnapshot);
backlog.fileProcessed("a.txt");
backlog.fileProcessed("b.txt");
Assert.assertEquals(1, backlog.getBacklog().size());
Assert.assertTrue(backlog.getBacklog().containsKey("c.txt"));
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(1, backlog.getBacklog().size());
Assert.assertTrue(backlog.getBacklog().containsKey("c.txt"));
backlog.fileProcessed("c.txt");
Assert.assertTrue(backlog.getBacklog().isEmpty());
backlog.processSnapshot(remoteSnapshot);
Assert.assertTrue(backlog.getBacklog().isEmpty());
}
@Test
public void testOneFileChangedSize() {
Assert.assertTrue(backlog.getBacklog().isEmpty());
Map<String, FileInfo> remoteSnapshot = generateInitialSnapshot();
backlog.processSnapshot(remoteSnapshot);
backlog.fileProcessed("a.txt");
backlog.fileProcessed("b.txt");
Assert.assertEquals(1, backlog.getBacklog().size());
Assert.assertTrue(backlog.getBacklog().containsKey("c.txt"));
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(1, backlog.getBacklog().size());
Assert.assertTrue(backlog.getBacklog().containsKey("c.txt"));
backlog.fileProcessed("c.txt");
Assert.assertTrue(backlog.getBacklog().isEmpty());
backlog.processSnapshot(remoteSnapshot);
remoteSnapshot.put("c.txt", new FileInfo("c.txt", 1001, 112));
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(1, backlog.getBacklog().size());
Assert.assertTrue(backlog.getBacklog().containsKey("c.txt"));
}
@Test
public void testOneFileChangedDate() {
Assert.assertTrue(backlog.getBacklog().isEmpty());
Map<String, FileInfo> remoteSnapshot = generateInitialSnapshot();
backlog.processSnapshot(remoteSnapshot);
backlog.fileProcessed("a.txt");
backlog.fileProcessed("b.txt");
Assert.assertEquals(1, backlog.getBacklog().size());
Assert.assertTrue(backlog.getBacklog().containsKey("c.txt"));
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(1, backlog.getBacklog().size());
Assert.assertTrue(backlog.getBacklog().containsKey("c.txt"));
backlog.fileProcessed("c.txt");
Assert.assertTrue(backlog.getBacklog().isEmpty());
backlog.processSnapshot(remoteSnapshot);
remoteSnapshot.put("c.txt", new FileInfo("c.txt", 1011, 102));
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(1, backlog.getBacklog().size());
Assert.assertTrue(backlog.getBacklog().containsKey("c.txt"));
}
@Test
public void testOneFileAdded() {
Assert.assertTrue(backlog.getBacklog().isEmpty());
Map<String, FileInfo> remoteSnapshot = generateInitialSnapshot();
backlog.processSnapshot(remoteSnapshot);
backlog.fileProcessed("a.txt");
backlog.fileProcessed("b.txt");
backlog.fileProcessed("c.txt");
Assert.assertTrue(backlog.getBacklog().isEmpty());
backlog.processSnapshot(remoteSnapshot);
remoteSnapshot.put("d.txt", new FileInfo("d.txt", 1003, 103));
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(1, backlog.getBacklog().size());
Assert.assertTrue(backlog.getBacklog().containsKey("d.txt"));
}
@Test
public void testOneFileRemoved() {
Assert.assertTrue(backlog.getBacklog().isEmpty());
Map<String, FileInfo> remoteSnapshot = generateInitialSnapshot();
backlog.processSnapshot(remoteSnapshot);
backlog.fileProcessed("a.txt");
backlog.fileProcessed("b.txt");
backlog.fileProcessed("c.txt");
Assert.assertTrue(backlog.getBacklog().isEmpty());
backlog.processSnapshot(remoteSnapshot);
remoteSnapshot.remove("c.txt");
backlog.processSnapshot(remoteSnapshot);
Assert.assertTrue(backlog.getBacklog().isEmpty());
}
@Test
public void testOneFileRemovedBeforeBeingProcessedInTheNextStep() {
Assert.assertTrue(backlog.getBacklog().isEmpty());
Map<String, FileInfo> remoteSnapshot = generateInitialSnapshot();
backlog.processSnapshot(remoteSnapshot);
Assert.assertTrue(backlog.getBacklog().containsKey("c.txt"));
remoteSnapshot.remove("c.txt");
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(2, backlog.getBacklog().size());
backlog.processSnapshot(remoteSnapshot);
Assert.assertEquals(2, backlog.getBacklog().size());
}
private static Map<String, FileInfo> generateInitialSnapshot() {
Map<String, FileInfo> remoteSnapshot = new HashMap<String, FileInfo>();
remoteSnapshot.put("a.txt", new FileInfo("a.txt", 1000, 100));
remoteSnapshot.put("b.txt", new FileInfo("b.txt", 1001, 101));
remoteSnapshot.put("c.txt", new FileInfo("c.txt", 1002, 102));
return remoteSnapshot;
}
}

View File

@@ -16,7 +16,15 @@
package org.springframework.integration.adapter.ftp;
import static org.easymock.classextension.EasyMock.*;
import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.getCurrentArguments;
import static org.easymock.EasyMock.isA;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.createNiceMock;
import static org.easymock.classextension.EasyMock.replay;
import static org.easymock.classextension.EasyMock.reset;
import static org.easymock.classextension.EasyMock.verify;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -37,7 +45,6 @@ import org.easymock.IAnswer;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
import org.springframework.integration.message.GenericMessage;
import org.springframework.integration.message.Message;
import org.springframework.integration.message.MessageCreator;
@@ -67,7 +74,6 @@ public class FtpSourceTests {
private Long size = 100l;
@Before
public void initializeFtpSource() {
ftpSource = new FtpSource(messageCreator, ftpClientPool);
@@ -78,7 +84,6 @@ public class FtpSourceTests {
reset(globalMocks);
}
@Test
public void retrieveSingleFile() throws Exception {
@@ -192,9 +197,9 @@ public class FtpSourceTests {
new File("test3") })));
}
@Test(timeout=60000)
@Test(timeout = 6000)
public void concurrentPollingSunnyDay() throws Exception {
final CountDownLatch recorded = new CountDownLatch(1);
this.ftpSource.setMaxFilesPerMessage(2);
// first run
FTPFile[] mockedFTPFiles = mockedFTPFilesNamed("test1", "test2", "test3", "test4", "test5");
@@ -215,17 +220,21 @@ public class FtpSourceTests {
return new GenericMessage(getCurrentArguments()[0]);
}
}).times(3);
replay(globalMocks);
recorded.countDown();
final CountDownLatch receivesDone = new CountDownLatch(3);
for (int i = 0; i < 3; i++) {
new Thread(new Runnable() {
public void run() {
Message<List<File>> recievedFiles = ftpSource.receive();
receivesDone.countDown();
Message<List<File>> recievedFiles = null;
try {
// make sure receive happens after recording
recorded.await();
recievedFiles = ftpSource.receive();
receivesDone.countDown();
//make sure onSend happens after all receives
receivesDone.await();
}
catch (InterruptedException e) {