Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

OPEN - issue BATCH-789: Remove mark/reset from ItemReader

Browse Source 
Shift chunk iteration into StepHandler - had to sacrifice interrupted check inside chunk
This commit is contained in:
dsyer
2008-08-20 15:24:05 +00:00
parent dbc24626f3
commit ef4f2900cc
19 changed files with 524 additions and 637 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/ItemOrientedStepHandlerTests.java
View File

@@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals;
import java.util.List;
import org.junit.Before;
import org.junit.Test;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobInstance;
@@ -32,6 +33,8 @@ import org.springframework.batch.item.ParseException;
import org.springframework.batch.item.UnexpectedInputException;
import org.springframework.batch.item.support.AbstractItemReader;
import org.springframework.batch.item.support.PassthroughItemProcessor;
import org.springframework.batch.repeat.policy.SimpleCompletionPolicy;
import org.springframework.batch.repeat.support.RepeatTemplate;
/**
* @author Dave Syer
@@ -42,22 +45,18 @@ public class ItemOrientedStepHandlerTests {
private StubItemReader itemReader = new StubItemReader();
private StubItemWriter itemWriter = new StubItemWriter();
private RepeatTemplate repeatTemplate = new RepeatTemplate();
@Before
public void setUp() {
repeatTemplate.setCompletionPolicy(new SimpleCompletionPolicy(2));
}
@Test
public void testHandle() throws Exception {
ItemOrientedStepHandler<String, String> handler = new ItemOrientedStepHandler<String, String>(itemReader,
new PassthroughItemProcessor<String>(), itemWriter);
StepContribution contribution = new StepContribution(new StepExecution("foo", new JobExecution(new JobInstance(
123L, new JobParameters(), "job"))));
handler.handle(contribution);
assertEquals(1, itemReader.count);
assertEquals("1", itemWriter.values);
}
@Test
public void testHandleCompositeItem() throws Exception {
ItemOrientedStepHandler<String, String> handler = new ItemOrientedStepHandler<String, String>(itemReader,
new AgrgegateItemProcessor(), itemWriter);
new PassthroughItemProcessor<String>(), itemWriter, repeatTemplate);
StepContribution contribution = new StepContribution(new StepExecution("foo", new JobExecution(new JobInstance(
123L, new JobParameters(), "job"))));
handler.handle(contribution);
@@ -65,6 +64,17 @@ public class ItemOrientedStepHandlerTests {
assertEquals("12", itemWriter.values);
}
@Test
public void testHandleCompositeItem() throws Exception {
ItemOrientedStepHandler<String, String> handler = new ItemOrientedStepHandler<String, String>(itemReader,
new AgrgegateItemProcessor(), itemWriter, repeatTemplate);
StepContribution contribution = new StepContribution(new StepExecution("foo", new JobExecution(new JobInstance(
123L, new JobParameters(), "job"))));
handler.handle(contribution);
assertEquals(4, itemReader.count);
assertEquals("1234", itemWriter.values);
}
/**
* @author Dave Syer
*

2
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/RepeatOperationsStepFactoryBeanTests.java
View File

@@ -39,7 +39,7 @@ import org.springframework.batch.support.transaction.ResourcelessTransactionMana
*/
public class RepeatOperationsStepFactoryBeanTests extends TestCase {
private RepeatOperationsStepFactoryBean<String,String> factory = new RepeatOperationsStepFactoryBean<String,String>();
private SimpleStepFactoryBean<String,String> factory = new SimpleStepFactoryBean<String,String>();
private List<String> list;

67
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/SimpleStepFactoryBeanTests.java
View File

@@ -16,13 +16,18 @@
package org.springframework.batch.core.step.item;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import junit.framework.TestCase;
import org.junit.Before;
import org.junit.Test;
import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.ChunkListener;
import org.springframework.batch.core.JobExecution;
@@ -53,7 +58,7 @@ import org.springframework.core.task.SimpleAsyncTaskExecutor;
/**
* Tests for {@link SimpleStepFactoryBean}.
*/
public class SimpleStepFactoryBeanTests extends TestCase {
public class SimpleStepFactoryBeanTests {
private List<Exception> recovered = new ArrayList<Exception>();
@@ -76,37 +81,15 @@ public class SimpleStepFactoryBeanTests extends TestCase {
}
};
protected void setUp() throws Exception {
super.setUp();
@Before
public void setUp() throws Exception {
job.setJobRepository(repository);
MapJobInstanceDao.clear();
MapJobExecutionDao.clear();
MapStepExecutionDao.clear();
}
private SimpleStepFactoryBean<String,String> getStepFactory(String arg) throws Exception {
return getStepFactory(new String[] { arg });
}
private SimpleStepFactoryBean<String,String> getStepFactory(String arg0, String arg1) throws Exception {
return getStepFactory(new String[] { arg0, arg1 });
}
private SimpleStepFactoryBean<String,String> getStepFactory(String[] args) throws Exception {
SimpleStepFactoryBean<String,String> factory = new SimpleStepFactoryBean<String,String>();
List<String> items = TransactionAwareProxyFactory.createTransactionalList();
items.addAll(Arrays.asList(args));
reader = new ListItemReader<String>(items);
factory.setItemReader(reader);
factory.setItemWriter(writer);
factory.setJobRepository(repository);
factory.setTransactionManager(new ResourcelessTransactionManager());
factory.setBeanName("stepName");
return factory;
}
@Test
public void testSimpleJob() throws Exception {
job.setSteps(new ArrayList<Step>());
@@ -125,6 +108,7 @@ public class SimpleStepFactoryBeanTests extends TestCase {
assertTrue(written.contains("foo"));
}
@Test
public void testSimpleConcurrentJob() throws Exception {
job.setSteps(new ArrayList<Step>());
@@ -144,12 +128,13 @@ public class SimpleStepFactoryBeanTests extends TestCase {
assertTrue(written.contains("foo"));
}
@Test
public void testSimpleJobWithItemListeners() throws Exception {
final List<Throwable> throwables = new ArrayList<Throwable>();
RepeatTemplate chunkOperations = new RepeatTemplate();
// Always handle the exception a check it is the right one...
// Always handle the exception to check it is the right one...
chunkOperations.setExceptionHandler(new ExceptionHandler() {
public void handleException(RepeatContext context, Throwable throwable) throws RuntimeException {
throwables.add(throwable);
@@ -179,8 +164,8 @@ public class SimpleStepFactoryBeanTests extends TestCase {
}
} });
factory.setChunkOperations(chunkOperations);
StepHandlerStep step = (StepHandlerStep) factory.getObject();
step.setChunkOperations(chunkOperations);
job.setSteps(Collections.singletonList((Step) step));
@@ -194,6 +179,7 @@ public class SimpleStepFactoryBeanTests extends TestCase {
assertEquals(3, recovered.size());
}
@Test
public void testExceptionTerminates() throws Exception {
SimpleStepFactoryBean<String,String> factory = getStepFactory(new String[] { "foo", "bar", "spam" });
factory.setBeanName("exceptionStep");
@@ -217,6 +203,7 @@ public class SimpleStepFactoryBeanTests extends TestCase {
assertEquals(BatchStatus.FAILED, jobExecution.getStatus());
}
@Test
public void testExceptionHandler() throws Exception {
SimpleStepFactoryBean<String,String> factory = getStepFactory(new String[] { "foo", "bar", "spam" });
factory.setBeanName("exceptionStep");
@@ -240,6 +227,7 @@ public class SimpleStepFactoryBeanTests extends TestCase {
assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus());
}
@Test
public void testChunkListeners() throws Exception {
String[] items = new String[] { "1", "2", "3", "4", "5", "6", "7" };
int commitInterval = 3;
@@ -282,11 +270,13 @@ public class SimpleStepFactoryBeanTests extends TestCase {
* Commit interval specified is not allowed to be zero or negative.
* @throws Exception
*/
@Test
public void testCommitIntervalMustBeGreaterThanZero() throws Exception {
SimpleStepFactoryBean<String,String> factory = getStepFactory("foo");
// nothing wrong here
factory.getObject();
factory = getStepFactory("foo");
// but exception expected after setting commit interval to value < 0
factory.setCommitInterval(-1);
try {
@@ -302,6 +292,7 @@ public class SimpleStepFactoryBeanTests extends TestCase {
* Commit interval specified is not allowed to be zero or negative.
* @throws Exception
*/
@Test
public void testCommitIntervalAndCompletionPolicyBothSet() throws Exception {
SimpleStepFactoryBean<String,String> factory = getStepFactory("foo");
@@ -318,4 +309,20 @@ public class SimpleStepFactoryBeanTests extends TestCase {
}
}
private SimpleStepFactoryBean<String,String> getStepFactory(String... args) throws Exception {
SimpleStepFactoryBean<String,String> factory = new SimpleStepFactoryBean<String,String>();
List<String> items = TransactionAwareProxyFactory.createTransactionalList();
items.addAll(Arrays.asList(args));
reader = new ListItemReader<String>(items);
factory.setItemReader(reader);
factory.setItemWriter(writer);
factory.setJobRepository(repository);
factory.setTransactionManager(new ResourcelessTransactionManager());
factory.setBeanName("stepName");
return factory;
}
}

60
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/SimpleStepHandler.java Normal file
View File

@@ -0,0 +1,60 @@
/*
* Copyright 2006-2007 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.core.step.item;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.support.PassthroughItemProcessor;
import org.springframework.batch.repeat.RepeatOperations;
import org.springframework.batch.repeat.policy.SimpleCompletionPolicy;
import org.springframework.batch.repeat.support.RepeatTemplate;
/**
* Simplest possible implementation of {@link StepHandler} with no skipping or
* recovering or processing. Just delegates all calls to the provided
* {@link ItemReader} and {@link ItemWriter}.
*
* @author Dave Syer
*/
public class SimpleStepHandler<T> extends ItemOrientedStepHandler<T, T> {
/**
*
*/
private static final RepeatTemplate repeatTemplate = new RepeatTemplate();
static {
// It's only for testing, and we don't want any infinite loops...
repeatTemplate.setCompletionPolicy(new SimpleCompletionPolicy(6));
}
/**
* Creates a {@link PassthroughItemProcessor} and uses it to create an
* instance of {@link ItemOrientedStepHandler}.
*/
public SimpleStepHandler(ItemReader<T> itemReader, ItemWriter<T> itemWriter) {
super(itemReader, new PassthroughItemProcessor<T>(), itemWriter, repeatTemplate);
}
/**
* Creates a {@link PassthroughItemProcessor} and uses it to create an
* instance of {@link ItemOrientedStepHandler}.
*/
public SimpleStepHandler(ItemReader<T> itemReader, ItemWriter<T> itemWriter, RepeatOperations repeatOperations) {
super(itemReader, new PassthroughItemProcessor<T>(), itemWriter, repeatOperations);
}
}

2
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/StatefulRetryStepFactoryBeanTests.java
View File

@@ -136,7 +136,7 @@ public class StatefulRetryStepFactoryBeanTests extends TestCase {
factory.setItemReader(provider);
factory.setRetryLimit(10);
factory.setSkippableExceptionClasses(new Class[0]);
AbstractStep step = (AbstractStep) factory.getObject();
Step step = (Step) factory.getObject();
StepExecution stepExecution = new StepExecution(step.getName(), jobExecution);
step.execute(stepExecution);

54
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/StepExecutorInterruptionTests.java
View File

@@ -68,18 +68,33 @@ public class StepExecutorInterruptionTests extends TestCase {
public void write(List<? extends Object> item) throws Exception {
}
};
step.setItemHandler(new SimpleStepHandler<Object>(new AbstractItemReader<Object>() {
public Object read() throws Exception {
return null;
}
}, itemWriter));
stepExecution = new StepExecution(step.getName(), jobExecution);
}
public void testInterruptChunk() throws Exception {
public void testInterruptStep() throws Exception {
Thread processingThread = createThread(stepExecution);
RepeatTemplate template = new RepeatTemplate();
// N.B, If we don't set the completion policy it might run forever
template.setCompletionPolicy(new SimpleCompletionPolicy(2));
step.setItemHandler(new SimpleStepHandler<Object>(new AbstractItemReader<Object>() {
public Object read() throws Exception {
// do something non-trivial (and not Thread.sleep())
double foo = 1;
for (int i = 2; i < 250; i++) {
foo = foo * i;
}
if (foo != 1) {
return new Double(foo);
}
else {
return null;
}
}
}, itemWriter, template));
processingThread.start();
Thread.sleep(100);
processingThread.interrupt();
@@ -96,14 +111,6 @@ public class StepExecutorInterruptionTests extends TestCase {
}
public void testInterruptStep() throws Exception {
RepeatTemplate template = new RepeatTemplate();
// N.B, If we don't set the completion policy it might run forever
template.setCompletionPolicy(new SimpleCompletionPolicy(2));
step.setChunkOperations(template);
testInterruptChunk();
}
public void testInterruptOnInterruptedException() throws Exception {
Thread processingThread = createThread(stepExecution);
@@ -176,23 +183,6 @@ public class StepExecutorInterruptionTests extends TestCase {
* @return
*/
private Thread createThread(final StepExecution stepExecution) {
step.setItemHandler(new SimpleStepHandler<Object>(new AbstractItemReader<Object>() {
public Object read() throws Exception {
// do something non-trivial (and not Thread.sleep())
double foo = 1;
for (int i = 2; i < 250; i++) {
foo = foo * i;
}
if (foo != 1) {
return new Double(foo);
}
else {
return null;
}
}
}, itemWriter));
Thread processingThread = new Thread() {
public void run() {
try {

12
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/StepHandlerStepIntegrationTests.java
View File

@@ -72,6 +72,8 @@ public class StepHandlerStepIntegrationTests {
private JobRepository jobRepository;
private RepeatTemplate chunkOperations;
private ItemReader<String> getReader(String[] args) {
return new ListItemReader<String>(Arrays.asList(args));
}
@@ -88,19 +90,17 @@ public class StepHandlerStepIntegrationTests {
jobRepositoryFactoryBean.setTransactionManager(transactionManager);
jobRepositoryFactoryBean.afterPropertiesSet();
jobRepository = (JobRepository) jobRepositoryFactoryBean.getObject();
RepeatTemplate template;
step = new StepHandlerStep("stepName");
step.setJobRepository(jobRepository);
step.setTransactionManager(transactionManager);
template = new RepeatTemplate();
RepeatTemplate template = new RepeatTemplate();
template.setCompletionPolicy(new SimpleCompletionPolicy(1));
step.setStepOperations(template);
// Only process one item:
template = new RepeatTemplate();
template.setCompletionPolicy(new SimpleCompletionPolicy(1));
step.setChunkOperations(template);
chunkOperations = new RepeatTemplate();
chunkOperations.setCompletionPolicy(new SimpleCompletionPolicy(1));
job = new JobSupport("FOO");
@@ -121,7 +121,7 @@ public class StepHandlerStepIntegrationTests {
}
});
}
}));
}, chunkOperations));
JobExecution jobExecution = jobRepository.createJobExecution(job, new JobParameters());
StepExecution stepExecution = new StepExecution(step.getName(), jobExecution);

39
spring-batch-core/src/test/java/org/springframework/batch/core/step/item/StepHandlerStepTests.java
View File

@@ -29,7 +29,6 @@ import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobInstance;
import org.springframework.batch.core.JobInterruptedException;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.StepExecutionListener;
import org.springframework.batch.core.UnexpectedJobExecutionException;
@@ -41,7 +40,6 @@ import org.springframework.batch.core.repository.dao.MapJobExecutionDao;
import org.springframework.batch.core.repository.dao.MapJobInstanceDao;
import org.springframework.batch.core.repository.dao.MapStepExecutionDao;
import org.springframework.batch.core.repository.support.SimpleJobRepository;
import org.springframework.batch.core.step.AbstractStep;
import org.springframework.batch.core.step.JobRepositorySupport;
import org.springframework.batch.core.step.StepInterruptionPolicy;
import org.springframework.batch.item.ExecutionContext;
@@ -93,9 +91,12 @@ public class StepHandlerStepTests extends TestCase {
return new ListItemReader<String>(Arrays.asList(args));
}
private AbstractStep getStep(String[] strings) throws Exception {
private StepHandlerStep getStep(String[] strings) throws Exception {
StepHandlerStep step = new StepHandlerStep("stepName");
step.setItemHandler(new SimpleStepHandler<String>(getReader(strings), itemWriter));
// Only process one item:
RepeatTemplate template = new RepeatTemplate();
template.setCompletionPolicy(new SimpleCompletionPolicy(1));
step.setItemHandler(new SimpleStepHandler<String>(getReader(strings), itemWriter, template));
step.setJobRepository(new JobRepositorySupport());
step.setTransactionManager(transactionManager);
return step;
@@ -108,16 +109,11 @@ public class StepHandlerStepTests extends TestCase {
transactionManager = new ResourcelessTransactionManager();
RepeatTemplate template;
RepeatTemplate template = new RepeatTemplate();
template.setCompletionPolicy(new SimpleCompletionPolicy(1));
step = (StepHandlerStep) getStep(new String[] { "foo", "bar", "spam" });
template = new RepeatTemplate();
template.setCompletionPolicy(new SimpleCompletionPolicy(1));
step = getStep(new String[] { "foo", "bar", "spam" });
step.setStepOperations(template);
// Only process one item:
template = new RepeatTemplate();
template.setCompletionPolicy(new SimpleCompletionPolicy(1));
step.setChunkOperations(template);
job = new JobSupport("FOO");
jobInstance = new JobInstance(new Long(0), new JobParameters(), job.getName());
@@ -181,25 +177,6 @@ public class StepHandlerStepTests extends TestCase {
}
public void testChunkExecutor() throws Exception {
RepeatTemplate template = new RepeatTemplate();
// Only process one item:
template.setCompletionPolicy(new SimpleCompletionPolicy(1));
step.setChunkOperations(template);
JobExecution jobExecution = new JobExecution(jobInstance);
StepExecution stepExecution = new StepExecution(step.getName(), jobExecution);
StepContribution contribution = stepExecution.createStepContribution();
step.processChunk(stepExecution, contribution);
assertEquals(1, processed.size());
assertEquals(0, stepExecution.getItemCount());
assertEquals(1, contribution.getItemCount());
}
public void testRepository() throws Exception {
SimpleJobRepository repository = new SimpleJobRepository(new MapJobInstanceDao(), new MapJobExecutionDao(),