diff --git a/spring-batch-core/src/main/java/org/springframework/batch/core/step/item/ChunkOrientedTasklet.java b/spring-batch-core/src/main/java/org/springframework/batch/core/step/item/ChunkOrientedTasklet.java index e944f31f3..1efd3a3e2 100644 --- a/spring-batch-core/src/main/java/org/springframework/batch/core/step/item/ChunkOrientedTasklet.java +++ b/spring-batch-core/src/main/java/org/springframework/batch/core/step/item/ChunkOrientedTasklet.java @@ -15,8 +15,13 @@ */ package org.springframework.batch.core.step.item; +import java.util.Collections; + +import org.springframework.batch.core.SkipListener; import org.springframework.batch.core.StepContribution; -import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.core.step.skip.ItemSkipPolicy; +import org.springframework.batch.core.step.skip.SkipLimitExceededException; +import org.springframework.batch.core.step.skip.SkipListenerFailedException; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.ItemWriter; @@ -24,16 +29,24 @@ import org.springframework.batch.repeat.ExitStatus; import org.springframework.batch.repeat.RepeatCallback; import org.springframework.batch.repeat.RepeatContext; import org.springframework.batch.repeat.RepeatOperations; +import org.springframework.batch.retry.RecoveryCallback; +import org.springframework.batch.retry.RetryCallback; +import org.springframework.batch.retry.RetryContext; +import org.springframework.batch.retry.RetryException; +import org.springframework.batch.retry.RetryOperations; +import org.springframework.batch.retry.support.DefaultRetryState; +import org.springframework.batch.support.Classifier; import org.springframework.core.AttributeAccessor; /** - * Simplest possible implementation of {@link Tasklet} with no skipping or - * recovering. Just delegates all calls to the provided {@link ItemReader} and - * {@link ItemWriter}. - * - * Provides extension points by protected {@link #read(StepContribution)} and - * {@link #write(Chunk, StepContribution)} methods that can be overriden to - * provide more sophisticated behaviour (e.g. skipping). + * If there is an exception on input it is skipped if allowed. If there is + * an exception on output, it will be re-thrown in any case, and the + * behaviour when the item is next encountered depends on the retryable and + * skippable exception configuration. If the exception is retryable the + * write will be attempted again up to the retry limit. When retry attempts + * are exhausted the skip listener is invoked and the skip count + * incremented. A retryable exception is thus also effectively also + * implicitly skippable. * * @author Dave Syer * @author Robert Kasanicky @@ -45,18 +58,30 @@ public class ChunkOrientedTasklet extends AbstractItemProcessingTasklet rollbackClassifier; + + public ChunkOrientedTasklet(ItemReader itemReader, ItemProcessor itemProcessor, ItemWriter itemWriter, - RepeatOperations repeatOperations) { + RepeatOperations chunkOperations, RetryOperations retryTemplate, + Classifier rollbackClassifier, ItemSkipPolicy readSkipPolicy, + ItemSkipPolicy writeSkipPolicy, ItemSkipPolicy processSkipPolicy) { super(itemReader, itemProcessor, itemWriter); - this.repeatOperations = repeatOperations; + this.repeatOperations = chunkOperations; + this.retryOperations = retryTemplate; + this.rollbackClassifier = rollbackClassifier; + this.readSkipPolicy = readSkipPolicy; + this.writeSkipPolicy = writeSkipPolicy; + this.processSkipPolicy = processSkipPolicy; } /** @@ -121,11 +146,46 @@ public class ChunkOrientedTasklet extends AbstractItemProcessingTasklet extends AbstractItemProcessingTasklet inputs, Chunk outputs) throws Exception { + /** + * Incorporate retry into the item processor stage. + * + * @see org.springframework.batch.core.step.item.ChunkOrientedTasklet#process(org.springframework.batch.core.StepContribution, + * org.springframework.batch.core.step.item.Chunk, + * org.springframework.batch.core.step.item.Chunk) + */ + protected void process(final StepContribution contribution, final Chunk inputs, final Chunk outputs) + throws Exception { + int filtered = 0; - for (T item : inputs) { - - S output = doProcess(item); + + for (final Chunk.ChunkIterator iterator = inputs.iterator(); iterator.hasNext();) { + + final T item = iterator.next(); + + RetryCallback retryCallback = new RetryCallback() { + + public S doWithRetry(RetryContext context) throws Exception { + S output = doProcess(item); + return output; + } + + }; + + RecoveryCallback recoveryCallback = new RecoveryCallback() { + + public S recover(RetryContext context) throws Exception { + Exception e = (Exception) context.getLastThrowable(); + if (processSkipPolicy.shouldSkip(e, contribution.getStepSkipCount())) { + contribution.incrementProcessSkipCount(); + iterator.remove(e); + return null; + } + else { + throw new RetryException("Non-skippable exception in recoverer while processing", e); + } + } + + }; + + S output = retryOperations.execute(retryCallback, recoveryCallback, new DefaultRetryState(item, rollbackClassifier)); if (output != null) { outputs.add(output); - } else { + } + else { filtered++; } + } + + for (ItemWrapper skip : inputs.getSkips()) { + Exception exception = skip.getException(); + try { + listener.onSkipInProcess(skip.getItem(), exception); + } + catch (RuntimeException e) { + throw new SkipListenerFailedException("Fatal exception in SkipListener.", e, exception); + } + } + contribution.incrementFilterCount(filtered); + inputs.clear(); + } /** + * Execute the business logic, delegating to the writer.
* - * @param chunk the items to write - * @param contribution current context + * Process the items with the {@link ItemWriter} in a stateful retry. + * Any {@link SkipListener} provided is called when retry attempts are + * exhausted. The listener callback (on write failure) will happen in + * the next transaction automatically.
*/ - protected void write(Chunk chunk, StepContribution contribution) throws Exception { - doWrite(chunk.getItems()); - contribution.incrementWriteCount(chunk.size()); + protected void write(final Chunk chunk, final StepContribution contribution) throws Exception { + + RetryCallback retryCallback = new RetryCallback() { + public Object doWithRetry(RetryContext context) throws Exception { + doWrite(chunk.getItems()); + contribution.incrementWriteCount(chunk.size()); + return null; + } + }; + + RecoveryCallback recoveryCallback = new RecoveryCallback() { + + public Object recover(RetryContext context) throws Exception { + + // small optimisation: if there was only one item, then we + // don't have to try writing it again to see if it fails... + if (chunk.size() == 1) { + Exception e = (Exception) context.getLastThrowable(); + checkSkipPolicy(contribution, chunk.iterator(), e); + return null; + } + + for (Chunk.ChunkIterator iterator = chunk.iterator(); iterator.hasNext();) { + S item = iterator.next(); + try { + doWrite(Collections.singletonList(item)); + contribution.incrementWriteCount(1); + } + catch (Exception e) { + checkSkipPolicy(contribution, iterator, e); + if (rollbackClassifier.classify(e)) { + throw e; + } + else { + logger.error("Exception encountered that does not classify for rollback: ", e); + } + } + } + + return null; + + } + + private void checkSkipPolicy(final StepContribution contribution, Chunk.ChunkIterator iterator, + Exception e) throws Exception { + if (writeSkipPolicy.shouldSkip(e, contribution.getStepSkipCount())) { + contribution.incrementWriteSkipCount(); + iterator.remove(e); + } + else { + throw new RetryException("Non-skippable exception in recoverer", e); + } + } + }; + + retryOperations.execute(retryCallback, recoveryCallback, new DefaultRetryState(chunk,rollbackClassifier)); + + for (ItemWrapper skip : chunk.getSkips()) { + Exception exception = skip.getException(); + try { + listener.onSkipInWrite(skip.getItem(), exception); + } + catch (RuntimeException e) { + throw new SkipListenerFailedException("Fatal exception in SkipListener.", e, exception); + } + } + chunk.clear(); + } + /** * @param attributes */ diff --git a/spring-batch-core/src/main/java/org/springframework/batch/core/step/item/SkipLimitStepFactoryBean.java b/spring-batch-core/src/main/java/org/springframework/batch/core/step/item/SkipLimitStepFactoryBean.java index 5c493c5c3..042be8fdb 100644 --- a/spring-batch-core/src/main/java/org/springframework/batch/core/step/item/SkipLimitStepFactoryBean.java +++ b/spring-batch-core/src/main/java/org/springframework/batch/core/step/item/SkipLimitStepFactoryBean.java @@ -2,36 +2,25 @@ package org.springframework.batch.core.step.item; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; -import org.springframework.batch.core.SkipListener; -import org.springframework.batch.core.StepContribution; import org.springframework.batch.core.step.skip.ItemSkipPolicy; import org.springframework.batch.core.step.skip.LimitCheckingItemSkipPolicy; import org.springframework.batch.core.step.skip.SkipLimitExceededException; import org.springframework.batch.core.step.skip.SkipListenerFailedException; import org.springframework.batch.core.step.tasklet.TaskletStep; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.ItemWriter; import org.springframework.batch.repeat.RepeatOperations; import org.springframework.batch.repeat.support.RepeatTemplate; -import org.springframework.batch.retry.RecoveryCallback; -import org.springframework.batch.retry.RetryCallback; -import org.springframework.batch.retry.RetryContext; import org.springframework.batch.retry.RetryException; import org.springframework.batch.retry.RetryListener; -import org.springframework.batch.retry.RetryOperations; import org.springframework.batch.retry.RetryPolicy; import org.springframework.batch.retry.backoff.BackOffPolicy; import org.springframework.batch.retry.policy.ExceptionClassifierRetryPolicy; import org.springframework.batch.retry.policy.MapRetryContextCache; import org.springframework.batch.retry.policy.RetryContextCache; import org.springframework.batch.retry.policy.SimpleRetryPolicy; -import org.springframework.batch.retry.support.DefaultRetryState; import org.springframework.batch.retry.support.RetryTemplate; import org.springframework.batch.support.Classifier; @@ -259,7 +248,7 @@ public class SkipLimitStepFactoryBean extends SimpleStepFactoryBean exceptions.addAll(new ArrayList>(retryableExceptionClasses)); ItemSkipPolicy writeSkipPolicy = new LimitCheckingItemSkipPolicy(skipLimit, exceptions, new ArrayList>(fatalExceptionClasses)); - ChunkOrientedTasklet tasklet = new StatefulRetryTasklet(getItemReader(), getItemProcessor(), + ChunkOrientedTasklet tasklet = new ChunkOrientedTasklet(getItemReader(), getItemProcessor(), getItemWriter(), getChunkOperations(), retryTemplate, rollbackClassifier, readSkipPolicy, writeSkipPolicy, writeSkipPolicy); tasklet.setListeners(getListeners()); @@ -281,240 +270,4 @@ public class SkipLimitStepFactoryBean extends SimpleStepFactoryBean fatalExceptionClasses = fatalExceptionList; } - /** - * If there is an exception on input it is skipped if allowed. If there is - * an exception on output, it will be re-thrown in any case, and the - * behaviour when the item is next encountered depends on the retryable and - * skippable exception configuration. If the exception is retryable the - * write will be attempted again up to the retry limit. When retry attempts - * are exhausted the skip listener is invoked and the skip count - * incremented. A retryable exception is thus also effectively also - * implicitly skippable. - * - * @author Dave Syer - * - */ - static class StatefulRetryTasklet extends ChunkOrientedTasklet { - - final private RetryOperations retryOperations; - - final private ItemSkipPolicy readSkipPolicy; - - final private ItemSkipPolicy writeSkipPolicy; - - final private ItemSkipPolicy processSkipPolicy; - - final private Classifier rollbackClassifier; - - /** - * @param itemReader - * @param itemWriter - * @param retryTemplate - */ - public StatefulRetryTasklet(ItemReader itemReader, - ItemProcessor itemProcessor, ItemWriter itemWriter, - RepeatOperations chunkOperations, RetryOperations retryTemplate, - Classifier rollbackClassifier, ItemSkipPolicy readSkipPolicy, - ItemSkipPolicy writeSkipPolicy, ItemSkipPolicy processSkipPolicy) { - super(itemReader, itemProcessor, itemWriter, chunkOperations); - this.retryOperations = retryTemplate; - this.rollbackClassifier = rollbackClassifier; - this.readSkipPolicy = readSkipPolicy; - this.writeSkipPolicy = writeSkipPolicy; - this.processSkipPolicy = processSkipPolicy; - } - - /** - * Tries to read the item from the reader, in case of exception skip the - * item if the skip policy allows, otherwise re-throw. - * - * @param contribution current StepContribution holding skipped items - * count - * @return next item for processing - */ - @Override - protected T read(StepContribution contribution) throws Exception { - - while (true) { - try { - return doRead(); - } - catch (Exception e) { - try { - if (readSkipPolicy.shouldSkip(e, contribution.getStepSkipCount())) { - // increment skip count and try again - contribution.incrementReadSkipCount(); - try { - listener.onSkipInRead(e); - } - catch (RuntimeException ex) { - throw new SkipListenerFailedException("Fatal exception in SkipListener.", ex, e); - } - logger.debug("Skipping failed input", e); - } - else { - // re-throw only when the skip policy runs out of - // patience - throw e; - } - } - catch (SkipLimitExceededException ex) { - // we are headed for a abnormal ending so bake in the - // skip count - throw ex; - } - } - } - - } - - /** - * Incorporate retry into the item processor stage. - * - * @see org.springframework.batch.core.step.item.ChunkOrientedTasklet#process(org.springframework.batch.core.StepContribution, - * org.springframework.batch.core.step.item.Chunk, - * org.springframework.batch.core.step.item.Chunk) - */ - @Override - protected void process(final StepContribution contribution, final Chunk inputs, final Chunk outputs) - throws Exception { - - int filtered = 0; - - for (final Chunk.ChunkIterator iterator = inputs.iterator(); iterator.hasNext();) { - - final T item = iterator.next(); - - RetryCallback retryCallback = new RetryCallback() { - - public S doWithRetry(RetryContext context) throws Exception { - S output = doProcess(item); - return output; - } - - }; - - RecoveryCallback recoveryCallback = new RecoveryCallback() { - - public S recover(RetryContext context) throws Exception { - Exception e = (Exception) context.getLastThrowable(); - if (processSkipPolicy.shouldSkip(e, contribution.getStepSkipCount())) { - contribution.incrementProcessSkipCount(); - iterator.remove(e); - return null; - } - else { - throw new RetryException("Non-skippable exception in recoverer while processing", e); - } - } - - }; - - S output = retryOperations.execute(retryCallback, recoveryCallback, new DefaultRetryState(item, rollbackClassifier)); - if (output != null) { - outputs.add(output); - } - else { - filtered++; - } - - } - - for (ItemWrapper skip : inputs.getSkips()) { - Exception exception = skip.getException(); - try { - listener.onSkipInProcess(skip.getItem(), exception); - } - catch (RuntimeException e) { - throw new SkipListenerFailedException("Fatal exception in SkipListener.", e, exception); - } - } - - contribution.incrementFilterCount(filtered); - - inputs.clear(); - - } - - /** - * Execute the business logic, delegating to the writer.
- * - * Process the items with the {@link ItemWriter} in a stateful retry. - * Any {@link SkipListener} provided is called when retry attempts are - * exhausted. The listener callback (on write failure) will happen in - * the next transaction automatically.
- */ - @Override - protected void write(final Chunk chunk, final StepContribution contribution) throws Exception { - - RetryCallback retryCallback = new RetryCallback() { - public Object doWithRetry(RetryContext context) throws Exception { - doWrite(chunk.getItems()); - contribution.incrementWriteCount(chunk.size()); - return null; - } - }; - - RecoveryCallback recoveryCallback = new RecoveryCallback() { - - public Object recover(RetryContext context) throws Exception { - - // small optimisation: if there was only one item, then we - // don't have to try writing it again to see if it fails... - if (chunk.size() == 1) { - Exception e = (Exception) context.getLastThrowable(); - checkSkipPolicy(contribution, chunk.iterator(), e); - return null; - } - - for (Chunk.ChunkIterator iterator = chunk.iterator(); iterator.hasNext();) { - S item = iterator.next(); - try { - doWrite(Collections.singletonList(item)); - contribution.incrementWriteCount(1); - } - catch (Exception e) { - checkSkipPolicy(contribution, iterator, e); - if (rollbackClassifier.classify(e)) { - throw e; - } - else { - logger.error("Exception encountered that does not classify for rollback: ", e); - } - } - } - - return null; - - } - - private void checkSkipPolicy(final StepContribution contribution, Chunk.ChunkIterator iterator, - Exception e) throws Exception { - if (writeSkipPolicy.shouldSkip(e, contribution.getStepSkipCount())) { - contribution.incrementWriteSkipCount(); - iterator.remove(e); - } - else { - throw new RetryException("Non-skippable exception in recoverer", e); - } - } - }; - - retryOperations.execute(retryCallback, recoveryCallback, new DefaultRetryState(chunk,rollbackClassifier)); - - for (ItemWrapper skip : chunk.getSkips()) { - Exception exception = skip.getException(); - try { - listener.onSkipInWrite(skip.getItem(), exception); - } - catch (RuntimeException e) { - throw new SkipListenerFailedException("Fatal exception in SkipListener.", e, exception); - } - } - - chunk.clear(); - - } - } - } diff --git a/spring-batch-core/src/test/java/org/springframework/batch/core/step/item/ChunkOrientedTaskletTests.java b/spring-batch-core/src/test/java/org/springframework/batch/core/step/item/ChunkOrientedTaskletTests.java index a9bc47632..b988f702f 100644 --- a/spring-batch-core/src/test/java/org/springframework/batch/core/step/item/ChunkOrientedTaskletTests.java +++ b/spring-batch-core/src/test/java/org/springframework/batch/core/step/item/ChunkOrientedTaskletTests.java @@ -61,7 +61,7 @@ public class ChunkOrientedTaskletTests { @Test public void testHandle() throws Exception { - ChunkOrientedTasklet handler = new ChunkOrientedTasklet(itemReader, + SimpleChunkOrientedTasklet handler = new SimpleChunkOrientedTasklet(itemReader, new PassthroughItemProcessor(), itemWriter, repeatTemplate); StepContribution contribution = new StepContribution(new StepExecution("foo", new JobExecution(new JobInstance( 123L, new JobParameters(), "job")))); @@ -75,7 +75,7 @@ public class ChunkOrientedTaskletTests { @Test public void testHandleWithItemProcessorFailure() throws Exception { - ChunkOrientedTasklet handler = new ChunkOrientedTasklet(itemReader, + SimpleChunkOrientedTasklet handler = new SimpleChunkOrientedTasklet(itemReader, new StubItemProcessor(), itemWriter, repeatTemplate); StepContribution contribution = new StepContribution(new StepExecution("foo", new JobExecution(new JobInstance( 123L, new JobParameters(), "job")))); @@ -95,7 +95,7 @@ public class ChunkOrientedTaskletTests { @Test public void testHandleCompositeItem() throws Exception { - ChunkOrientedTasklet handler = new ChunkOrientedTasklet(itemReader, + SimpleChunkOrientedTasklet handler = new SimpleChunkOrientedTasklet(itemReader, new AggregateItemProcessor(), itemWriter, repeatTemplate); StepContribution contribution = new StepContribution(new StepExecution("foo", new JobExecution(new JobInstance( 123L, new JobParameters(), "job")))); diff --git a/spring-batch-core/src/test/java/org/springframework/batch/core/step/item/StatefulRetryTaskletTests.java b/spring-batch-core/src/test/java/org/springframework/batch/core/step/item/StatefulRetryTaskletTests.java index dc7a0825f..6524d2a33 100644 --- a/spring-batch-core/src/test/java/org/springframework/batch/core/step/item/StatefulRetryTaskletTests.java +++ b/spring-batch-core/src/test/java/org/springframework/batch/core/step/item/StatefulRetryTaskletTests.java @@ -28,7 +28,6 @@ import org.junit.Before; import org.junit.Test; import org.springframework.batch.core.StepContribution; import org.springframework.batch.core.StepExecution; -import org.springframework.batch.core.step.item.SkipLimitStepFactoryBean.StatefulRetryTasklet; import org.springframework.batch.core.step.skip.ItemSkipPolicy; import org.springframework.batch.core.step.skip.SkipLimitExceededException; import org.springframework.batch.core.step.tasklet.BasicAttributeAccessor; @@ -62,7 +61,7 @@ public class StatefulRetryTaskletTests { private List processed = new ArrayList(); - private StatefulRetryTasklet handler; + private ChunkOrientedTasklet handler; private RepeatTemplate chunkOperations = new RepeatTemplate(); @@ -110,7 +109,7 @@ public class StatefulRetryTaskletTests { @Test public void testBasicHandle() throws Exception { - handler = new StatefulRetryTasklet(itemReader, itemProcessor, itemWriter, chunkOperations, + handler = new ChunkOrientedTasklet(itemReader, itemProcessor, itemWriter, chunkOperations, retryTemplate, rollbackClassifier, readSkipPolicy, writeSkipPolicy, writeSkipPolicy); StepContribution contribution = new StepExecution("foo", null).createStepContribution(); handler.execute(contribution, new BasicAttributeAccessor()); @@ -119,7 +118,7 @@ public class StatefulRetryTaskletTests { @Test public void testSkipOnRead() throws Exception { - handler = new StatefulRetryTasklet(new ItemReader() { + handler = new ChunkOrientedTasklet(new ItemReader() { public Integer read() throws Exception, UnexpectedInputException, NoWorkFoundException, ParseException { throw new RuntimeException("Barf!"); } @@ -141,7 +140,7 @@ public class StatefulRetryTaskletTests { @Test public void testSkipSingleItemOnWrite() throws Exception { - handler = new StatefulRetryTasklet(itemReader, itemProcessor, new ItemWriter() { + handler = new ChunkOrientedTasklet(itemReader, itemProcessor, new ItemWriter() { public void write(List items) throws Exception { written.addAll(items); throw new RuntimeException("Barf!"); @@ -166,7 +165,7 @@ public class StatefulRetryTaskletTests { @Test public void testSkipMultipleItemsOnWrite() throws Exception { - handler = new StatefulRetryTasklet(itemReader, itemProcessor, new ItemWriter() { + handler = new ChunkOrientedTasklet(itemReader, itemProcessor, new ItemWriter() { public void write(List items) throws Exception { logger.debug("Writing items: " + items); written.addAll(items); @@ -218,7 +217,7 @@ public class StatefulRetryTaskletTests { @Test public void testSkipSingleItemOnProcess() throws Exception { - handler = new StatefulRetryTasklet(itemReader, new ItemProcessor() { + handler = new ChunkOrientedTasklet(itemReader, new ItemProcessor() { public String process(Integer item) throws Exception { logger.debug("Processing item: " + item); processed.add(item); @@ -258,7 +257,7 @@ public class StatefulRetryTaskletTests { @Test public void testSkipOverLimitOnProcess() throws Exception { - handler = new StatefulRetryTasklet(itemReader, new ItemProcessor() { + handler = new ChunkOrientedTasklet(itemReader, new ItemProcessor() { public String process(Integer item) throws Exception { logger.debug("Processing item: " + item); processed.add(item); diff --git a/spring-batch-core/src/test/java/org/springframework/batch/core/step/tasklet/SimpleChunkOrientedTasklet.java b/spring-batch-core/src/test/java/org/springframework/batch/core/step/tasklet/SimpleChunkOrientedTasklet.java index 9398dae10..b44b51551 100644 --- a/spring-batch-core/src/test/java/org/springframework/batch/core/step/tasklet/SimpleChunkOrientedTasklet.java +++ b/spring-batch-core/src/test/java/org/springframework/batch/core/step/tasklet/SimpleChunkOrientedTasklet.java @@ -31,7 +31,7 @@ import org.springframework.batch.repeat.support.RepeatTemplate; * * @author Dave Syer */ -public class SimpleChunkOrientedTasklet extends ChunkOrientedTasklet { +public class SimpleChunkOrientedTasklet extends org.springframework.batch.core.step.item.SimpleChunkOrientedTasklet { /** *