Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add MessageChannelPartitionHandler

Browse Source
This commit is contained in:
dsyer
2009-05-06 12:22:21 +00:00
parent 11deb78a44
commit 91cf3c83c3
14 changed files with 687 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
spring-batch-integration/.settings/org.springframework.ide.eclipse.beans.core.prefs Normal file
View File

@@ -0,0 +1,3 @@
#Wed May 06 08:20:22 BST 2009
eclipse.preferences.version=1
org.springframework.ide.eclipse.beans.core.ignoreMissingNamespaceHandler=false

4
spring-batch-integration/.springBeans
View File

@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<beansProjectDescription>
<version>1</version>
<pluginVersion><![CDATA[2.2.0.v200809261800]]></pluginVersion>
<pluginVersion><![CDATA[2.2.3.RELEASE]]></pluginVersion>
<configSuffixes>
<configSuffix><![CDATA[xml]]></configSuffix>
</configSuffixes>
@@ -18,6 +18,8 @@
<config>src/test/resources/org/springframework/batch/integration/retry/RetryRepeatTransactionalPollingIntegrationTests-context.xml</config>
<config>src/test/resources/org/springframework/batch/integration/SmokeTests-context.xml</config>
<config>src/test/resources/org/springframework/batch/integration/retry/RetryTransactionalPollingIntegrationTests-context.xml</config>
<config>src/test/resources/org/springframework/batch/integration/item/MessagingGatewayIntegrationTests-context.xml</config>
<config>src/test/resources/org/springframework/batch/integration/partition/VanillaIntegrationTests-context.xml</config>
</configs>
<configSets>
</configSets>

47
spring-batch-integration/src/main/java/org/springframework/batch/integration/partition/BeanFactoryStepLocator.java Normal file
View File

@@ -0,0 +1,47 @@
package org.springframework.batch.integration.partition;
import java.util.Arrays;
import java.util.Collection;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.step.StepLocator;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.beans.factory.BeanFactoryAware;
import org.springframework.beans.factory.ListableBeanFactory;
import org.springframework.util.Assert;
/**
* A {@link StepLocator} implementation that just looks in its enclosing bean
* factory for components of type {@link Step}.
*
* @author Dave Syer
*
*/
public class BeanFactoryStepLocator implements StepLocator, BeanFactoryAware {
private BeanFactory beanFactory;
public void setBeanFactory(BeanFactory beanFactory) throws BeansException {
this.beanFactory = beanFactory;
}
/**
* Look up a bean with the provided name of type {@link Step}.
* @see StepLocator#getStep(String)
*/
public Step getStep(String stepName) {
return (Step) beanFactory.getBean(stepName, Step.class);
}
/**
* Look in the bean factory for all beans of type {@link Step}.
* @throws IllegalStateException if the {@link BeanFactory} is not listable
* @see StepLocator#getStepNames()
*/
public Collection<String> getStepNames() {
Assert.state(beanFactory instanceof ListableBeanFactory, "BeanFactory is not listable.");
return Arrays.asList(((ListableBeanFactory) beanFactory).getBeanNamesForType(Step.class));
}
}

138
spring-batch-integration/src/main/java/org/springframework/batch/integration/partition/MessageChannelPartitionHandler.java Normal file
View File

@@ -0,0 +1,138 @@
package org.springframework.batch.integration.partition;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.partition.PartitionHandler;
import org.springframework.batch.core.partition.StepExecutionSplitter;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.integration.annotation.Aggregator;
import org.springframework.integration.annotation.MessageEndpoint;
import org.springframework.integration.core.Message;
import org.springframework.integration.core.MessageChannel;
import org.springframework.integration.gateway.MessagingGateway;
import org.springframework.integration.message.MessageBuilder;
import org.springframework.util.Assert;
/**
* A {@link PartitionHandler} that uses {@link MessageChannel} instances to send
* instructions to remote workers and receive their responses. The
* {@link MessageChannel} provides a nice abstraction so that the location of
* the workers and the transport used to communicate with them can be changed at
* run time. The communication with the remote workers does not need to be
* transactional or have guaranteed delivery, so a local thread pool based
* implementation works as well as a remote web service or JMS implementation.
* If a remote worker fails or doesn't send a reply message, the job will fail
* and can be restarted to pick up missing messages and processing. The remote
* workers need access to the Spring Batch {@link JobRepository} so that the
* shared state across those restarts can be managed centrally.
*
* @author Dave Syer
*
*/
@MessageEndpoint
public class MessageChannelPartitionHandler implements PartitionHandler {
private int gridSize = 1;
private MessagingGateway messagingGateway;
private String stepName;
public void afterPropertiesSet() throws Exception {
Assert.notNull(stepName, "A step name must be provided for the remote workers.");
Assert.state(messagingGateway != null, "The MessagingGateway must be set");
}
/**
* A pre-configured gateway for sending and receiving messages to the remote
* workers. Using this property allows a large degree of control over the
* timeouts and other properties of the send. It should have channels set up
* internally:
* <ul>
* <li>request channel capable of accepting {@link StepExecutionRequest}
* payloads</li>
* <li>reply channel that returns a list of {@link StepExecution} results</li>
* </ul>
* The timeout for the repoy should be set sufficiently long that the remote
* steps have time to complete.
*
* @param messagingGateway the {@link MessagingGateway} to set
*/
public void setMessagingGateway(MessagingGateway messagingGateway) {
this.messagingGateway = messagingGateway;
}
/**
* Passed to the {@link StepExecutionSplitter} in the
* {@link #handle(StepExecutionSplitter, StepExecution)} method, instructing
* it how many {@link StepExecution} instances are required, ideally. The
* {@link StepExecutionSplitter} is allowed to ignore the grid size in the
* case of a restart, since the input data partitions must be preserved.
*
* @param gridSize the number of step executions that will be created
*/
public void setGridSize(int gridSize) {
this.gridSize = gridSize;
}
/**
* The name of the {@link Step} that will be used to execute the partitioned
* {@link StepExecution}. This is a regular Spring Batch step, with all the
* business logic required to complete an execution based on the input
* parameters in its {@link StepExecution} context. The name will be
* translated into a {@link Step} instance by the remote worker.
*
* @param stepName the name of the {@link Step} instance to execute business
* logic
*/
public void setStepName(String stepName) {
this.stepName = stepName;
}
/**
* @param messages the messages to be aggregated
* @return the list as it was passed in
*/
@Aggregator(sendPartialResultsOnTimeout = true)
public List<?> aggregate(List<?> messages) {
return messages;
}
/**
* Sends {@link StepExecutionRequest} objects to the request channel of the
* {@link MessagingGateway}, and then receives the result back as a list of
* {@link StepExecution} on a reply channel. Use the
* {@link #aggregate(List)} method as an aggregator of the individual remote
* replies. The receive timeout needs to be set realistically in the
* {@link MessagingGateway} <b>and</b> the aggregator, so that there is a
* good chance of all work being done.
*
* @see PartitionHandler#handle(StepExecutionSplitter, StepExecution)
*/
public Collection<StepExecution> handle(StepExecutionSplitter stepExecutionSplitter,
StepExecution masterStepExecution) throws Exception {
Set<StepExecution> split = stepExecutionSplitter.split(masterStepExecution, gridSize);
int count = 0;
for (StepExecution stepExecution : split) {
messagingGateway.send(createMessage(count++, split.size(), new StepExecutionRequest(stepName, stepExecution
.getJobExecutionId(), stepExecution.getId())));
}
@SuppressWarnings("unchecked")
Collection<StepExecution> result = (Collection<StepExecution>) messagingGateway.receive();
return result;
}
private Message<StepExecutionRequest> createMessage(int sequenceNumber, int sequenceSize,
StepExecutionRequest stepExecutionRequest) {
return MessageBuilder.withPayload(stepExecutionRequest).setSequenceNumber(sequenceNumber).setSequenceSize(
sequenceSize).setCorrelationId(
stepExecutionRequest.getJobExecutionId() + ":" + stepExecutionRequest.getStepName()).build();
}
}

37
spring-batch-integration/src/main/java/org/springframework/batch/integration/partition/StepExecutionRequest.java Normal file
View File

@@ -0,0 +1,37 @@
package org.springframework.batch.integration.partition;
import java.io.Serializable;
public class StepExecutionRequest implements Serializable {
private final Long stepExecutionId;
private final String stepName;
private final Long jobExecutionId;
public StepExecutionRequest(String stepName, Long jobExecutionId, Long stepExecutionId) {
this.stepName = stepName;
this.jobExecutionId = jobExecutionId;
this.stepExecutionId = stepExecutionId;
}
public Long getJobExecutionId() {
return jobExecutionId;
}
public Long getStepExecutionId() {
return stepExecutionId;
}
public String getStepName() {
return stepName;
}
@Override
public String toString() {
return String.format("StepExecutionRequest: [jobExecutionId=%d, stepExecutionId=%d, stepName=%s]",
jobExecutionId, stepExecutionId, stepName);
}
}

81
spring-batch-integration/src/main/java/org/springframework/batch/integration/partition/StepExecutionRequestHandler.java Normal file
View File

@@ -0,0 +1,81 @@
package org.springframework.batch.integration.partition;
import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.JobInterruptedException;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.step.StepLocator;
import org.springframework.integration.annotation.MessageEndpoint;
import org.springframework.integration.annotation.ServiceActivator;
/**
* A {@link MessageEndpoint} that can handle a {@link StepExecutionRequest} and
* return a {@link StepExecution} as the result. Typically these need to be
* aggregated into a response to a partition handler.
*
* @author Dave Syer
*
*/
@MessageEndpoint
public class StepExecutionRequestHandler {
private JobExplorer jobExplorer;
private StepLocator stepLocator;
/**
* Used to locate a {@link Step} to execute for each request.
* @param stepLocator a {@link StepLocator}
*/
public void setStepLocator(StepLocator stepLocator) {
this.stepLocator = stepLocator;
}
/**
* An explorer that should be used to check for {@link StepExecution}
* completion.
*
* @param jobExplorer a {@link JobExplorer} that is linked to the shared
* repository used by all remote workers.
*/
public void setJobExplorer(JobExplorer jobExplorer) {
this.jobExplorer = jobExplorer;
}
@ServiceActivator
public StepExecution handle(StepExecutionRequest request) {
Long jobExecutionId = request.getJobExecutionId();
Long stepExecutionId = request.getStepExecutionId();
StepExecution stepExecution = jobExplorer.getStepExecution(jobExecutionId, stepExecutionId);
if (stepExecution == null) {
// TODO: create new Exception
throw new IllegalStateException("No StepExecution could be located for this request: " + request);
}
String stepName = request.getStepName();
Step step = stepLocator.getStep(stepName);
if (step == null) {
// TODO: create new Exception
throw new IllegalStateException(String.format("No Step with name [%s] could be located.", stepName));
}
try {
step.execute(stepExecution);
}
catch (JobInterruptedException e) {
stepExecution.setStatus(BatchStatus.STOPPED);
// TODO: maybe update stepExecution in repository
}
catch (Throwable e) {
stepExecution.addFailureException(e);
stepExecution.setStatus(BatchStatus.FAILED);
// TODO: maybe update stepExecution in repository
}
return stepExecution;
}
}

59
spring-batch-integration/src/test/java/org/springframework/batch/integration/partition/BeanFactoryStepLocatorTests.java Normal file
View File

@@ -0,0 +1,59 @@
package org.springframework.batch.integration.partition;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import org.junit.Test;
import org.springframework.batch.core.JobInterruptedException;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.StepExecution;
import org.springframework.beans.factory.support.DefaultListableBeanFactory;
public class BeanFactoryStepLocatorTests {
private BeanFactoryStepLocator stepLocator = new BeanFactoryStepLocator();
private DefaultListableBeanFactory beanFactory = new DefaultListableBeanFactory();
@Test
public void testGetStep() throws Exception {
beanFactory.registerSingleton("foo", new StubStep("foo"));
stepLocator.setBeanFactory(beanFactory);
assertNotNull(stepLocator.getStep("foo"));
}
@Test
public void testGetStepNames() throws Exception {
beanFactory.registerSingleton("foo", new StubStep("foo"));
beanFactory.registerSingleton("bar", new StubStep("bar"));
stepLocator.setBeanFactory(beanFactory);
assertEquals(2, stepLocator.getStepNames().size());
}
private static final class StubStep implements Step {
private String name;
public StubStep(String name) {
this.name = name;
}
public void execute(StepExecution stepExecution) throws JobInterruptedException {
}
public String getName() {
return name;
}
public int getStartLimit() {
// TODO Auto-generated method stub
return 0;
}
public boolean isAllowStartIfComplete() {
// TODO Auto-generated method stub
return false;
}
}
}

57
spring-batch-integration/src/test/java/org/springframework/batch/integration/partition/ExampleItemReader.java Normal file
View File

@@ -0,0 +1,57 @@
package org.springframework.batch.integration.partition;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemStream;
import org.springframework.batch.item.ItemStreamException;
/**
* {@link ItemReader} with hard-coded input data.
*/
public class ExampleItemReader implements ItemReader<String>, ItemStream {
private Log logger = LogFactory.getLog(getClass());
private String[] input = { "Hello", "world!", "Go", "on", "punk", "make", "my", "day!" };
private int index = 0;
public static volatile boolean fail = false;
/**
* Reads next record from input
*/
public String read() throws Exception {
if (index >= input.length) {
return null;
}
logger.info(String.format("Processing input index=%s, item=%s, in (%s)", index, input[index], this));
if (fail && index == 4) {
synchronized (ExampleItemReader.class) {
if (fail) {
// Only fail once per flag setting...
fail = false;
logger.info(String.format("Throwing exception index=%s, item=%s, in (%s)", index, input[index],
this));
index++;
throw new RuntimeException("Planned failure");
}
}
}
return input[index++];
}
public void close() throws ItemStreamException {
}
public void open(ExecutionContext executionContext) throws ItemStreamException {
index = (int) executionContext.getLong("POSITION", 0);
}
public void update(ExecutionContext executionContext) throws ItemStreamException {
executionContext.putLong("POSITION", index);
}
}

70
spring-batch-integration/src/test/java/org/springframework/batch/integration/partition/ExampleItemReaderTests.java Normal file
View File

@@ -0,0 +1,70 @@
package org.springframework.batch.integration.partition;
import static org.junit.Assert.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.batch.item.ExecutionContext;
public class ExampleItemReaderTests {
private ExampleItemReader reader = new ExampleItemReader();
@Before
@After
public void ensureFailFlagUnset() {
ExampleItemReader.fail = false;
}
@Test
public void testRead() throws Exception {
int count = 0;
while (reader.read()!=null) {
count++;
}
assertEquals(8, count);
}
@Test
public void testOpen() throws Exception {
ExecutionContext context = new ExecutionContext();
for (int i=0; i<4; i++) {
reader.read();
}
reader.update(context);
reader.open(context);
int count = 0;
while (reader.read()!=null) {
count++;
}
assertEquals(4, count);
}
@Test
public void testFailAndRestart() throws Exception {
ExecutionContext context = new ExecutionContext();
ExampleItemReader.fail = true;
for (int i=0; i<4; i++) {
reader.read();
reader.update(context);
}
try {
reader.read();
reader.update(context);
fail("Expected Exception");
}
catch (Exception e) {
// expected
assertEquals("Planned failure", e.getMessage());
}
assertFalse(ExampleItemReader.fail);
reader.open(context);
int count = 0;
while (reader.read()!=null) {
count++;
}
assertEquals(4, count);
}
}

23
spring-batch-integration/src/test/java/org/springframework/batch/integration/partition/ExampleItemWriter.java Normal file
View File

@@ -0,0 +1,23 @@
package org.springframework.batch.integration.partition;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.batch.item.ItemWriter;
/**
* Dummy {@link ItemWriter} which only logs data it receives.
*/
public class ExampleItemWriter implements ItemWriter<Object> {
private static final Log log = LogFactory.getLog(ExampleItemWriter.class);
/**
* @see ItemWriter#write(List)
*/
public void write(List<? extends Object> data) throws Exception {
log.info(data);
}
}

70
spring-batch-integration/src/test/java/org/springframework/batch/integration/partition/VanillaIntegrationTests.java Normal file
View File

@@ -0,0 +1,70 @@
/*
* Copyright 2006-2007 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.integration.partition;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobInstance;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author Dave Syer
*
*/
@ContextConfiguration
@RunWith(SpringJUnit4ClassRunner.class)
public class VanillaIntegrationTests {
@Autowired
private JobLauncher jobLauncher;
@Autowired
private Job job;
@Autowired
private JobExplorer jobExplorer;
@Test
public void testSimpleProperties() throws Exception {
assertNotNull(jobLauncher);
}
@Test
public void testLaunchJob() throws Exception {
int before = jobExplorer.getJobInstances(job.getName(), 0, 100).size();
assertNotNull(jobLauncher.run(job, new JobParameters()));
List<JobInstance> jobInstances = jobExplorer.getJobInstances(job.getName(), 0, 100);
int after = jobInstances.size();
assertEquals(1, after-before);
JobExecution jobExecution = jobExplorer.getJobExecutions(jobInstances.get(jobInstances.size()-1)).get(0);
assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus());
assertEquals(3, jobExecution.getStepExecutions().size());
}
}

2
spring-batch-integration/src/test/resources/log4j.properties
View File

@@ -4,6 +4,6 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %5p %t [%c] - <%m>%n
log4j.logger.org.springframework.batch=DEBUG
log4j.logger.org.springframework.batch.core=DEBUG
log4j.category.org.springframework.integration=DEBUG
log4j.category.org.springframework.transaction=DEBUG

3
spring-batch-integration/src/test/resources/org/springframework/batch/integration/SmokeTests-context.xml
View File

@@ -10,10 +10,11 @@
http://www.springframework.org/schema/context/spring-context-2.5.xsd
http://www.springframework.org/schema/tx
http://www.springframework.org/schema/tx/spring-tx-2.5.xsd">
<annotation-config/>
<channel id="smokein"/>
<channel id="smokeout">
<queue/>
</channel>
</beans:beans>

96
spring-batch-integration/src/test/resources/org/springframework/batch/integration/partition/VanillaIntegrationTests-context.xml Normal file
View File

@@ -0,0 +1,96 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:p="http://www.springframework.org/schema/p" xmlns:integration="http://www.springframework.org/schema/integration"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration-1.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx.xsd">
<import resource="classpath:/simple-job-launcher-context.xml" />
<integration:channel id="requests">
<integration:queue />
</integration:channel>
<integration:channel id="staging">
<integration:queue />
</integration:channel>
<integration:channel id="replies">
<integration:queue />
</integration:channel>
<integration:service-activator ref="stepExecutionRequestHandler"
input-channel="requests" output-channel="staging">
<integration:poller>
<integration:interval-trigger interval="10"/>
</integration:poller>
</integration:service-activator>
<integration:aggregator ref="partitionHandler"
timeout="10000" input-channel="staging" output-channel="replies">
<integration:poller>
<integration:interval-trigger interval="10"/>
</integration:poller>
</integration:aggregator>
<!-- This is the "remote" worker (which in this case is local) -->
<bean id="stepExecutionRequestHandler"
class="org.springframework.batch.integration.partition.StepExecutionRequestHandler"
p:jobExplorer-ref="jobExplorer" p:stepLocator-ref="stepLocator" />
<bean id="stepLocator"
class="org.springframework.batch.integration.partition.BeanFactoryStepLocator" />
<bean id="jobExplorer"
class="org.springframework.batch.core.explore.support.MapJobExplorerFactoryBean" />
<bean id="partitionHandler"
class="org.springframework.batch.integration.partition.MessageChannelPartitionHandler">
<property name="messagingGateway">
<bean
class="org.springframework.integration.gateway.SimpleMessagingGateway">
<property name="requestChannel" ref="requests" />
<property name="replyChannel" ref="replies" />
<property name="replyTimeout" value="10000" />
</bean>
</property>
<property name="stepName" value="step1" />
<property name="gridSize" value="2" />
</bean>
<bean id="job1" parent="simpleJob">
<property name="steps">
<bean name="step1:master"
class="org.springframework.batch.core.partition.support.PartitionStep">
<property name="partitionHandler" ref="partitionHandler" />
<property name="stepExecutionSplitter">
<bean
class="org.springframework.batch.core.partition.support.SimpleStepExecutionSplitter">
<constructor-arg ref="jobRepository" />
<constructor-arg ref="step1" />
</bean>
</property>
<property name="jobRepository" ref="jobRepository" />
</bean>
</property>
</bean>
<bean id="step1" parent="simpleStep">
<property name="itemReader">
<bean
class="org.springframework.batch.integration.partition.ExampleItemReader"
scope="step">
<!--
<property name="resource"
value="#{stepAttributes[jobParameters['resource']]}"/>
-->
</bean>
</property>
<property name="itemWriter">
<bean
class="org.springframework.batch.integration.partition.ExampleItemWriter" />
</property>
</bean>
<bean class="org.springframework.batch.core.scope.StepScope" />
</beans>