Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

OPEN - issue BATCH-220: Chunk-oriented approach to processing

Browse Source 
Add small optimisation to skip chunk with single item
This commit is contained in:
dsyer
2008-08-28 07:56:38 +00:00
parent 31109f0a4d
commit 223a58da18
12 changed files with 647 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
spring-batch-core/src/test/java/org/springframework/batch/core/listener/CompositeItemProcessListenerTests.java Normal file
View File

@@ -0,0 +1,88 @@
/*
* Copyright 2006-2008 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.core.listener;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import java.util.ArrayList;
import org.junit.Before;
import org.junit.Test;
import org.springframework.batch.core.ItemProcessListener;
/**
* @author Dave Syer
*
*/
public class CompositeItemProcessListenerTests {
private ItemProcessListener<Object, Object> listener;
private CompositeItemProcessListener<Object, Object> compositeListener;
@SuppressWarnings("unchecked")
@Before
public void setUp() throws Exception {
listener = createMock(ItemProcessListener.class);
compositeListener = new CompositeItemProcessListener<Object, Object>();
compositeListener.register(listener);
}
@Test
public void testBeforeRProcess() {
Object item = new Object();
listener.beforeProcess(item);
replay(listener);
compositeListener.beforeProcess(item);
verify(listener);
}
@Test
public void testAfterRead() {
Object item = new Object();
Object result = new Object();
listener.afterProcess(item, result);
replay(listener);
compositeListener.afterProcess(item, result);
verify(listener);
}
@Test
public void testOnReadError() {
Object item = new Object();
Exception ex = new Exception();
listener.onProcessError(item, ex);
replay(listener);
compositeListener.onProcessError(item, ex);
verify(listener);
}
@Test
public void testSetListeners() throws Exception {
compositeListener.setListeners(new ArrayList<ItemProcessListener<? super Object, ? super Object>>() {
{
add(listener);
}
});
listener.beforeProcess(null);
replay(listener);
compositeListener.beforeProcess(null);
verify(listener);
}
}

24
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/StatefulRetryStepFactoryBeanTests.java
View File

@@ -73,7 +73,7 @@ public class StatefulRetryStepFactoryBeanTests {
private List<Object> provided = new ArrayList<Object>();
private List<Object> written = TransactionAwareProxyFactory.createTransactionalList();
int count = 0;
private SimpleJobRepository repository = new SimpleJobRepository(new MapJobInstanceDao(), new MapJobExecutionDao(),
@@ -260,8 +260,8 @@ public class StatefulRetryStepFactoryBeanTests {
// [a, b, c, d, e, f, null]
assertEquals(7, provided.size());
// [a, b, b, b, b, b, b, c, d, d, d, d, d, d, e, f]
assertEquals(16, processed.size());
// [a, b, b, b, b, b, c, d, d, d, d, d, e, f]
assertEquals(14, processed.size());
// [b, d]
assertEquals(2, recovered.size());
}
@@ -324,7 +324,8 @@ public class StatefulRetryStepFactoryBeanTests {
// [a, b, c, d, e, f, null]
assertEquals(7, provided.size());
// [a, b, c, a, b, c, a, b, c, a, b, c, a, b, c, a, b, a, c, d, e, f, d, e, f, d, e, f, d, e, f, d, e, f, d, e, f]
// [a, b, c, a, b, c, a, b, c, a, b, c, a, b, c, a, b, a, c, d, e, f, d,
// e, f, d, e, f, d, e, f, d, e, f, d, e, f]
assertEquals(37, processed.size());
// [b, d]
assertEquals(2, recovered.size());
@@ -374,9 +375,10 @@ public class StatefulRetryStepFactoryBeanTests {
assertEquals(0, stepExecution.getSkipCount());
// [b]
assertEquals(1, provided.size());
// the failed items are tried one more time than the limit (TODO: maybe fix this?)
// [b, b, b, b, b]
assertEquals(5, processed.size());
// the failed items are tried up to the limit (but only precisely so if
// the commit interval is 1)
// [b, b, b, b]
assertEquals(4, processed.size());
// []
assertEquals(0, recovered.size());
assertEquals(1, stepExecution.getItemCount());
@@ -432,8 +434,8 @@ public class StatefulRetryStepFactoryBeanTests {
assertEquals(0, stepExecution.getSkipCount());
// [b]
assertEquals(1, provided.size());
// [b, b]
assertEquals(2, processed.size());
// [b]
assertEquals(1, processed.size());
// []
assertEquals(0, recovered.size());
assertEquals(1, stepExecution.getItemCount());
@@ -478,8 +480,8 @@ public class StatefulRetryStepFactoryBeanTests {
assertEquals(0, stepExecution.getSkipCount());
// [b]
assertEquals(1, provided.size());
// [b, b, b, b, b]
assertEquals(5, processed.size());
// [b, b, b, b]
assertEquals(4, processed.size());
// []
assertEquals(0, recovered.size());
assertEquals(1, stepExecution.getItemCount());

208
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/StatefulRetryStepHandlerTests.java Normal file
View File

@@ -0,0 +1,208 @@
/*
* Copyright 2006-2007 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.core.step.item;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.junit.Before;
import org.junit.Test;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.step.handler.BasicAttributeAccessor;
import org.springframework.batch.core.step.item.SkipLimitStepFactoryBean.StatefulRetryStepHandler;
import org.springframework.batch.core.step.skip.ItemSkipPolicy;
import org.springframework.batch.core.step.skip.SkipLimitExceededException;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.NoWorkFoundException;
import org.springframework.batch.item.ParseException;
import org.springframework.batch.item.UnexpectedInputException;
import org.springframework.batch.repeat.policy.SimpleCompletionPolicy;
import org.springframework.batch.repeat.support.RepeatTemplate;
import org.springframework.batch.retry.policy.NeverRetryPolicy;
import org.springframework.batch.retry.support.RetryTemplate;
/**
* @author Dave Syer
*
*/
public class StatefulRetryStepHandlerTests {
private Log logger = LogFactory.getLog(getClass());
private int count = 0;
private int limit = 3;
protected int skipLimit = 2;
protected List<String> written = new ArrayList<String>();
private StatefulRetryStepHandler<Integer, String> handler;
private RepeatTemplate chunkOperations = new RepeatTemplate();
private ItemReader<Integer> itemReader = new ItemReader<Integer>() {
public Integer read() {
return count++ >= limit ? null : count;
};
};
private ItemWriter<String> itemWriter = new ItemWriter<String>() {
public void write(List<? extends String> items) throws Exception {
written.addAll(items);
}
};
private ItemProcessor<Integer, String> itemProcessor = new ItemProcessor<Integer, String>() {
public String process(Integer item) throws Exception {
return "" + item;
}
};
private RetryTemplate retryTemplate = new RetryTemplate();
private ItemSkipPolicy readSkipPolicy = new ItemSkipPolicy() {
public boolean shouldSkip(Throwable t, int skipCount) throws SkipLimitExceededException {
if (skipCount < skipLimit) {
return true;
}
throw new SkipLimitExceededException(skipLimit, t);
}
};
private ItemSkipPolicy writeSkipPolicy = readSkipPolicy;
@Before
public void setUp() {
retryTemplate.setRetryPolicy(new NeverRetryPolicy());
}
@Test
public void testBasicHandle() throws Exception {
handler = new StatefulRetryStepHandler<Integer, String>(itemReader, itemProcessor, itemWriter, chunkOperations,
retryTemplate, readSkipPolicy, writeSkipPolicy);
StepContribution contribution = new StepExecution("foo", null).createStepContribution();
handler.handle(contribution, new BasicAttributeAccessor());
assertEquals(limit, contribution.getItemCount());
}
@Test
public void testSkipOnRead() throws Exception {
handler = new StatefulRetryStepHandler<Integer, String>(new ItemReader<Integer>() {
public Integer read() throws Exception, UnexpectedInputException, NoWorkFoundException, ParseException {
throw new RuntimeException("Barf!");
}
}, itemProcessor, itemWriter, chunkOperations, retryTemplate, readSkipPolicy, writeSkipPolicy);
chunkOperations.setCompletionPolicy(new SimpleCompletionPolicy(1));
StepContribution contribution = new StepExecution("foo", null).createStepContribution();
BasicAttributeAccessor attributes = new BasicAttributeAccessor();
try {
handler.handle(contribution, attributes);
fail("Expected SkipLimitExceededException");
}
catch (SkipLimitExceededException e) {
// expected
}
assertEquals(0, contribution.getItemCount());
assertEquals(2, contribution.getReadSkipCount());
}
@Test
public void testSkipSingleItemOnWrite() throws Exception {
handler = new StatefulRetryStepHandler<Integer, String>(itemReader, itemProcessor, new ItemWriter<String>() {
public void write(List<? extends String> items) throws Exception {
written.addAll(items);
throw new RuntimeException("Barf!");
}
}, chunkOperations, retryTemplate, readSkipPolicy, writeSkipPolicy);
chunkOperations.setCompletionPolicy(new SimpleCompletionPolicy(1));
StepContribution contribution = new StepExecution("foo", null).createStepContribution();
BasicAttributeAccessor attributes = new BasicAttributeAccessor();
try {
handler.handle(contribution, attributes);
fail("Expected RuntimeException");
}
catch (Exception e) {
assertEquals("Barf!", e.getMessage());
}
assertTrue(attributes.hasAttribute("OUTPUT_BUFFER_KEY"));
handler.handle(contribution, attributes);
assertEquals(1, contribution.getItemCount());
assertEquals(1, contribution.getWriteSkipCount());
assertEquals(1, written.size());
}
@Test
public void testSkipMultipleItems() throws Exception {
handler = new StatefulRetryStepHandler<Integer, String>(itemReader, itemProcessor, new ItemWriter<String>() {
public void write(List<? extends String> items) throws Exception {
logger.debug("Writing items: "+items);
written.addAll(items);
throw new RuntimeException("Barf!");
}
}, chunkOperations, retryTemplate, readSkipPolicy, writeSkipPolicy);
chunkOperations.setCompletionPolicy(new SimpleCompletionPolicy(2));
StepContribution contribution = new StepExecution("foo", null).createStepContribution();
BasicAttributeAccessor attributes = new BasicAttributeAccessor();
// Count to 3: (try + skip + skip)
for (int i = 0; i < 3; i++) {
try {
handler.handle(contribution, attributes);
fail("Expected RuntimeException on i="+i);
}
catch (Exception e) {
assertEquals("Barf!", e.getMessage());
}
assertTrue(attributes.hasAttribute("OUTPUT_BUFFER_KEY"));
}
@SuppressWarnings("unchecked")
Chunk<String> chunk = (Chunk<String>) attributes.getAttribute("OUTPUT_BUFFER_KEY");
assertEquals(1, chunk.getSkips().size());
// The last recovery for this chunk...
handler.handle(contribution, attributes);
attributes = new BasicAttributeAccessor();
try {
handler.handle(contribution, attributes);
fail("Expected RuntimeException on i=");
}
catch (Exception e) {
assertEquals("Barf!", e.getMessage());
}
try {
handler.handle(contribution, attributes);
fail("Expected SkipLimitExceededException");
}
catch (SkipLimitExceededException e) {
// expected
}
assertTrue(attributes.hasAttribute("OUTPUT_BUFFER_KEY"));
assertEquals(3, contribution.getItemCount());
assertEquals(2, contribution.getWriteSkipCount());
assertEquals(5, written.size());
}
}