Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

OPEN - issue BATCH-220: Chunk-oriented approach to processing

Browse Source 
Skip should be working now for reads and writes (not processing).
This commit is contained in:
dsyer
2008-08-27 11:08:47 +00:00
parent b55fc3b475
commit 32118407c7
11 changed files with 276 additions and 490 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/AlmostStatefulRetryChunkTests.java
View File

@@ -37,10 +37,6 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class AlmostStatefulRetryChunkTests {
private enum CallType {
RUN, RETRY;
}
private Log logger = LogFactory.getLog(getClass());
private final Chunk<String> chunk;
@@ -53,8 +49,6 @@ public class AlmostStatefulRetryChunkTests {
private int count = 0;
private Object lastCallType;
public AlmostStatefulRetryChunkTests(String[] args, int limit) {
chunk = new Chunk<String>();
for (String string : args) {
@@ -67,88 +61,78 @@ public class AlmostStatefulRetryChunkTests {
public void testRetry() throws Exception {
logger.debug("Starting simple scenario");
List<String> items = new ArrayList<String>(chunk.getItems());
int before = items.size();
items.removeAll(Collections.singleton("fail"));
boolean error = true;
while (error && count++ < BACKSTOP_LIMIT) {
try {
if (!chunk.canRetry()) {
// success
logger.debug("Run items: " + chunk.getItems());
lastCallType = CallType.RUN;
runChunk(chunk);
}
else {
logger.debug(String.format("Retry (attempts=%d) items: %s", retryAttempts, chunk.getItems()));
lastCallType = CallType.RETRY;
try {
retryChunk(chunk);
}
catch (Exception e) {
chunk.rethrow(e);
}
}
chunk.rethrow();
statefulRetry(chunk);
error = false;
}
catch (Exception e) {
error = true;
}
}
logger.debug("Items: " + chunk.getItems());
logger.debug("Chunk: " + chunk);
assertTrue("Backstop reached. Probably an infinite loop...", count < BACKSTOP_LIMIT);
assertEquals(CallType.RETRY, lastCallType);
assertFalse(chunk.getItems().contains("fail"));
assertEquals(items, chunk.getItems());
assertEquals(before-chunk.getItems().size(), chunk.getSkips().size());
}
/**
* @param chunk
* @throws Exception
*/
private void retryChunk(Chunk<String> chunk) throws Exception {
try {
// N.B. a classic stateful retry goes straight to recovery here
doWrite(chunk);
retryAttempts = 0;
}
catch (Exception e) {
if (++retryAttempts > retryLimit) {
// recovery
private void statefulRetry(Chunk<String> chunk) throws Exception {
if (retryAttempts <= retryLimit) {
try {
// N.B. a classic stateful retry goes straight to recovery here
logger.debug(String.format("Retry (attempts=%d) chunk: %s", retryAttempts, chunk));
doWrite(chunk.getItems());
retryAttempts = 0;
if (chunk.canSkip()) {
chunk.getSkippedItem();
}
else {
throw e;
}
}
else {
catch (Exception e) {
retryAttempts++;
// stateful retry always rethrow
throw e;
}
}
else {
try {
logger.debug(String.format("Recover (attempts=%d) chunk: %s", retryAttempts, chunk));
recover(chunk);
}
finally {
retryAttempts = 0;
}
}
// recovery
return;
}
/**
* @param chunk
* @throws Exception
*/
private void recover(Chunk<String> chunk) throws Exception {
for (Chunk<String>.ChunkIterator iterator = chunk.iterator(); iterator.hasNext();) {
String string = iterator.next();
try {
doWrite(Collections.singletonList(string));
} catch (Exception e) {
iterator.remove(e);
throw e;
}
}
}
/**
* @param chunk
* @throws Exception
*/
private void runChunk(Chunk<String> chunk) throws Exception {
try {
doWrite(chunk);
}
catch (Exception e) {
chunk.rethrow(e);
}
}
/**
* @param chunk
* @throws Exception
*/
private void doWrite(Chunk<String> chunk) throws Exception {
List<String> items = chunk.getItems();
private void doWrite(List<String> items) throws Exception {
if (items.contains("fail")) {
throw new Exception();
}

42
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/SkipLimitStepFactoryBeanTests.java
View File

@@ -93,7 +93,7 @@ public class SkipLimitStepFactoryBeanTests {
// only write exception caused rollback, but more than once because it
// has to go back and split the chunk up to isolate the failed item
assertEquals(3, stepExecution.getRollbackCount());
assertEquals(2, stepExecution.getRollbackCount());
// writer did not skip "2" as it never made it to writer, only "4" did
assertTrue(reader.processed.contains("4"));
@@ -229,8 +229,8 @@ public class SkipLimitStepFactoryBeanTests {
assertFalse(reader.processed.contains("2"));
assertTrue(reader.processed.contains("4"));
// "1" was sent to writer but never comitted
List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray(""));
// only "1" was ever committed
List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray("1"));
assertEquals(expectedOutput, writer.written);
}
@@ -333,16 +333,14 @@ public class SkipLimitStepFactoryBeanTests {
StepExecution stepExecution = jobExecution.createStepExecution(step);
// TODO: uncomment this!
// step.execute(stepExecution);
// assertEquals(4, stepExecution.getSkipCount());
// assertEquals(3, stepExecution.getReadSkipCount());
// assertEquals(1, stepExecution.getWriteSkipCount());
//
// // skipped 2,3,4,5
// List<String> expectedOutput =
// Arrays.asList(StringUtils.commaDelimitedListToStringArray("1,6"));
// assertEquals(expectedOutput, writer.written);
step.execute(stepExecution);
assertEquals(4, stepExecution.getSkipCount());
assertEquals(3, stepExecution.getReadSkipCount());
assertEquals(1, stepExecution.getWriteSkipCount());
// skipped 2,3,4,5
List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray("1,6"));
assertEquals(expectedOutput, writer.written);
}
@@ -366,16 +364,14 @@ public class SkipLimitStepFactoryBeanTests {
StepExecution stepExecution = jobExecution.createStepExecution(step);
// TODO: uncomment this!
// step.execute(stepExecution);
// assertEquals(4, stepExecution.getSkipCount());
// assertEquals(2, stepExecution.getReadSkipCount());
// assertEquals(2, stepExecution.getWriteSkipCount());
//
// // skipped 2,3,4,5
// List<String> expectedOutput =
// Arrays.asList(StringUtils.commaDelimitedListToStringArray("1,6,7"));
// assertEquals(expectedOutput, writer.written);
step.execute(stepExecution);
assertEquals(4, stepExecution.getSkipCount());
assertEquals(2, stepExecution.getReadSkipCount());
assertEquals(2, stepExecution.getWriteSkipCount());
// skipped 2,3,4,5
List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray("1,6,7"));
assertEquals(expectedOutput, writer.written);
}

66
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/StatefulRetryStepFactoryBeanTests.java
View File

@@ -52,7 +52,9 @@ import org.springframework.batch.retry.policy.MapRetryContextCache;
import org.springframework.batch.retry.policy.RetryCacheCapacityExceededException;
import org.springframework.batch.retry.policy.SimpleRetryPolicy;
import org.springframework.batch.support.transaction.ResourcelessTransactionManager;
import org.springframework.batch.support.transaction.TransactionAwareProxyFactory;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.util.StringUtils;
/**
* @author Dave Syer
@@ -70,6 +72,8 @@ public class StatefulRetryStepFactoryBeanTests {
private List<Object> provided = new ArrayList<Object>();
private List<Object> written = TransactionAwareProxyFactory.createTransactionalList();
int count = 0;
private SimpleJobRepository repository = new SimpleJobRepository(new MapJobInstanceDao(), new MapJobExecutionDao(),
@@ -137,8 +141,7 @@ public class StatefulRetryStepFactoryBeanTests {
*/
@Test
public void testSuccessfulRetryWithReadFailure() throws Exception {
List<String> items = Arrays.asList(new String[] { "a", "b", "c" });
ItemReader<String> provider = new ListItemReader<String>(items) {
ItemReader<String> provider = new ListItemReader<String>(Arrays.asList("a", "b", "c")) {
public String read() {
String item = super.read();
provided.add(item);
@@ -177,8 +180,7 @@ public class StatefulRetryStepFactoryBeanTests {
}
});
factory.setSkipLimit(2);
List<String> items = Arrays.asList(new String[] { "a", "b", "c", "d", "e", "f" });
ItemReader<String> provider = new ListItemReader<String>(items) {
ItemReader<String> provider = new ListItemReader<String>(Arrays.asList("a", "b", "c", "d", "e", "f")) {
public String read() {
String item = super.read();
count++;
@@ -216,8 +218,7 @@ public class StatefulRetryStepFactoryBeanTests {
}
} });
factory.setSkipLimit(2);
List<String> items = Arrays.asList(new String[] { "a", "b", "c", "d", "e", "f" });
ItemReader<String> provider = new ListItemReader<String>(items) {
ItemReader<String> provider = new ListItemReader<String>(Arrays.asList("a", "b", "c", "d", "e", "f")) {
public String read() {
String item = super.read();
logger.debug("Read Called! Item: [" + item + "]");
@@ -231,6 +232,7 @@ public class StatefulRetryStepFactoryBeanTests {
public void write(List<? extends String> item) throws Exception {
logger.debug("Write Called! Item: [" + item + "]");
processed.addAll(item);
written.addAll(item);
if (item.contains("b") || item.contains("d")) {
throw new RuntimeException("Write error - planned but recoverable.");
}
@@ -253,10 +255,13 @@ public class StatefulRetryStepFactoryBeanTests {
assertEquals(2, stepExecution.getSkipCount());
assertEquals(2, stepExecution.getWriteSkipCount());
List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray("a,c,e,f"));
assertEquals(expectedOutput, written);
// [a, b, c, d, e, f, null]
assertEquals(7, provided.size());
// [a, b, b, b, b, b, c, d, d, d, d, d, e, f]
assertEquals(14, processed.size());
// [a, b, b, b, b, b, b, c, d, d, d, d, d, d, e, f]
assertEquals(16, processed.size());
// [b, d]
assertEquals(2, recovered.size());
}
@@ -277,8 +282,7 @@ public class StatefulRetryStepFactoryBeanTests {
}
} });
factory.setSkipLimit(2);
List<String> items = Arrays.asList(new String[] { "a", "b", "c", "d", "e", "f" });
ItemReader<String> provider = new ListItemReader<String>(items) {
ItemReader<String> provider = new ListItemReader<String>(Arrays.asList("a", "b", "c", "d", "e", "f")) {
public String read() {
String item = super.read();
logger.debug("Read Called! Item: [" + item + "]");
@@ -292,6 +296,7 @@ public class StatefulRetryStepFactoryBeanTests {
public void write(List<? extends String> item) throws Exception {
logger.debug("Write Called! Item: [" + item + "]");
processed.addAll(item);
written.addAll(item);
if (item.contains("b") || item.contains("d")) {
throw new RuntimeException("Write error - planned but recoverable.");
}
@@ -314,10 +319,13 @@ public class StatefulRetryStepFactoryBeanTests {
assertEquals(2, stepExecution.getSkipCount());
assertEquals(2, stepExecution.getWriteSkipCount());
List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray("a,c,e,f"));
assertEquals(expectedOutput, written);
// [a, b, c, d, e, f, null]
assertEquals(7, provided.size());
// [a, b, c, a, b, c, b, b, b, b, b, c, a, c, d, e, f, d, d, d, d, e, f, e, f]
assertEquals(25, processed.size());
// [a, b, c, a, b, c, a, b, c, a, b, c, a, b, c, a, b, a, c, d, e, f, d, e, f, d, e, f, d, e, f, d, e, f, d, e, f]
assertEquals(37, processed.size());
// [b, d]
assertEquals(2, recovered.size());
}
@@ -331,8 +339,7 @@ public class StatefulRetryStepFactoryBeanTests {
});
factory.setRetryLimit(4);
factory.setSkipLimit(0);
List<String> items = Arrays.asList(new String[] { "b" });
ItemReader<String> provider = new ListItemReader<String>(items) {
ItemReader<String> provider = new ListItemReader<String>(Arrays.asList("b")) {
public String read() {
String item = super.read();
provided.add(item);
@@ -343,6 +350,7 @@ public class StatefulRetryStepFactoryBeanTests {
ItemWriter<String> itemWriter = new ItemWriter<String>() {
public void write(List<? extends String> item) throws Exception {
processed.addAll(item);
written.addAll(item);
logger.debug("Write Called! Item: [" + item + "]");
throw new RuntimeException("Write error - planned but retryable.");
}
@@ -360,11 +368,15 @@ public class StatefulRetryStepFactoryBeanTests {
// expected
}
List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray(""));
assertEquals(expectedOutput, written);
assertEquals(0, stepExecution.getSkipCount());
// [b]
assertEquals(1, provided.size());
// [b, b, b, b]
assertEquals(4, processed.size());
// the failed items are tried one more time than the limit (TODO: maybe fix this?)
// [b, b, b, b, b]
assertEquals(5, processed.size());
// []
assertEquals(0, recovered.size());
assertEquals(1, stepExecution.getItemCount());
@@ -383,8 +395,7 @@ public class StatefulRetryStepFactoryBeanTests {
factory.setRetryableExceptionClasses(new HashSet<Class<? extends Throwable>>());
factory.setSkipLimit(1);
List<String> items = Arrays.asList(new String[] { "b" });
ItemReader<String> provider = new ListItemReader<String>(items) {
ItemReader<String> provider = new ListItemReader<String>(Arrays.asList("b")) {
public String read() {
String item = super.read();
provided.add(item);
@@ -395,6 +406,7 @@ public class StatefulRetryStepFactoryBeanTests {
ItemWriter<String> itemWriter = new ItemWriter<String>() {
public void write(List<? extends String> item) throws Exception {
processed.addAll(item);
written.addAll(item);
logger.debug("Write Called! Item: [" + item + "]");
throw new RuntimeException("Write error - planned but not skippable.");
}
@@ -414,11 +426,14 @@ public class StatefulRetryStepFactoryBeanTests {
assertTrue("Wrong message: " + message, message.contains("Write error - planned but not skippable."));
}
List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray(""));
assertEquals(expectedOutput, written);
assertEquals(0, stepExecution.getSkipCount());
// [b]
assertEquals(1, provided.size());
// [b]
assertEquals(1, processed.size());
// [b, b]
assertEquals(2, processed.size());
// []
assertEquals(0, recovered.size());
assertEquals(1, stepExecution.getItemCount());
@@ -428,8 +443,7 @@ public class StatefulRetryStepFactoryBeanTests {
public void testRetryPolicy() throws Exception {
factory.setRetryPolicy(new SimpleRetryPolicy(4));
factory.setSkipLimit(0);
List<String> items = Arrays.asList(new String[] { "b" });
ItemReader<String> provider = new ListItemReader<String>(items) {
ItemReader<String> provider = new ListItemReader<String>(Arrays.asList("b")) {
public String read() {
String item = super.read();
provided.add(item);
@@ -440,6 +454,7 @@ public class StatefulRetryStepFactoryBeanTests {
ItemWriter<String> itemWriter = new ItemWriter<String>() {
public void write(List<? extends String> item) throws Exception {
processed.addAll(item);
written.addAll(item);
logger.debug("Write Called! Item: [" + item + "]");
throw new RuntimeException("Write error - planned but retryable.");
}
@@ -457,11 +472,14 @@ public class StatefulRetryStepFactoryBeanTests {
// expected
}
List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray(""));
assertEquals(expectedOutput, written);
assertEquals(0, stepExecution.getSkipCount());
// [b]
assertEquals(1, provided.size());
// [b, b, b, b]
assertEquals(4, processed.size());
// [b, b, b, b, b]
assertEquals(5, processed.size());
// []
assertEquals(0, recovered.size());
assertEquals(1, stepExecution.getItemCount());

159
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/StatelessRetryChunkTests.java
View File

@@ -1,159 +0,0 @@
/*
* Copyright 2006-2007 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.core.step.item;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
/**
* @author Dave Syer
*
*/
@RunWith(Parameterized.class)
public class StatelessRetryChunkTests {
private Log logger = LogFactory.getLog(getClass());
private final Chunk<String> chunk;
private static final int BACKSTOP_LIMIT = 1000;
private int count = 0;
public StatelessRetryChunkTests(String[] args) {
chunk = new Chunk<String>();
for (String string : args) {
chunk.add(string);
}
}
@Test
public void testRetry() throws Exception {
logger.debug("Starting simple scenario");
List<String> items = new ArrayList<String>(chunk.getItems());
int before = items.size();
items.removeAll(Collections.singleton("fail"));
int errors = 0;
boolean error = true;
while (error && count++ < BACKSTOP_LIMIT) {
try {
// success
logger.debug("Run items: " + chunk.getItems());
retryChunk(chunk);
error = false;
}
catch (SpecialException e) {
error = true;
}
catch (Exception e) {
errors++;
error = true;
}
}
logger.debug("Items: " + chunk.getItems());
assertTrue("Backstop reached. Probably an infinite loop...", count < BACKSTOP_LIMIT);
assertFalse(chunk.getItems().contains("fail"));
assertEquals(items, chunk.getItems());
int after = chunk.getItems().size();
logger.debug(String.format("Error count: %d, size before: %d, size after: %d", errors, before, after));
}
/**
* @param chunk
* @throws Exception
*/
private void retryChunk(Chunk<String> chunk) throws Exception {
boolean complete = chunk.isComplete();
try {
doWrite(chunk);
}
catch (Exception e) {
if (chunk.canSkip()) {
chunk.getSkippedItem();
}
else {
if (complete) {
chunk.rethrow(e);
} else {
chunk.rethrow(new SpecialException());
}
}
}
chunk.rethrow();
}
/**
* @param chunk
* @throws Exception
*/
private void doWrite(Chunk<String> chunk) throws Exception {
List<String> items = chunk.getItems();
if (items.contains("fail")) {
throw new Exception();
}
}
@Parameters
public static List<Object[]> data() {
List<Object[]> params = new ArrayList<Object[]>();
params.add(new Object[] { new String[] { "foo" } });
params.add(new Object[] { new String[] { "foo", "bar" } });
params.add(new Object[] { new String[] { "foo", "bar", "spam" } });
params.add(new Object[] { new String[] { "foo", "bar", "spam", "maps", "rab", "oof" } });
params.add(new Object[] { new String[] { "fail" } });
params.add(new Object[] { new String[] { "foo", "fail" } });
params.add(new Object[] { new String[] { "fail", "bar" } });
params.add(new Object[] { new String[] { "foo", "fail", "spam" } });
params.add(new Object[] { new String[] { "fail", "bar", "spam" } });
params.add(new Object[] { new String[] { "foo", "fail", "spam", "maps", "rab", "oof" } });
params.add(new Object[] { new String[] { "foo", "fail", "spam", "fail", "rab", "oof" } });
params.add(new Object[] { new String[] { "fail", "bar", "spam", "fail", "rab", "oof" } });
params.add(new Object[] { new String[] { "foo", "fail", "fail", "fail", "rab", "oof" } });
params.add(new Object[] { new String[] { "fail" } });
params.add(new Object[] { new String[] { "foo", "fail", "fail", "fail", "rab", "oof" } });
params.add(new Object[] { new String[] { "foo", "fail", "fail", "fail", "rab", "oof" } });
return params;
}
/**
* @author Dave Syer
*
*/
public class SpecialException extends Exception {
}
}