Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add builders to simplify remote partitioning setup

Browse Source 
Resolves BATCH-2687

* use hard coded constants in docs
* change visibility of method `beanFactory` to public in builders
This commit is contained in:
Mahmoud Ben Hassine
2018-07-05 22:21:48 +02:00
committed by Michael Minella
parent 9f2b752f3f
commit 39676a45c4
16 changed files with 1351 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
spring-batch-samples/src/main/java/org/springframework/batch/sample/remotepartitioning/aggregating/MasterConfiguration.java
View File

@@ -21,22 +21,15 @@ import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.partition.PartitionHandler;
import org.springframework.batch.core.partition.support.Partitioner;
import org.springframework.batch.integration.partition.MessageChannelPartitionHandler;
import org.springframework.batch.integration.config.annotation.EnableBatchIntegration;
import org.springframework.batch.integration.partition.RemotePartitioningMasterStepBuilderFactory;
import org.springframework.batch.sample.remotepartitioning.BasicPartitioner;
import org.springframework.batch.sample.remotepartitioning.BrokerConfiguration;
import org.springframework.batch.sample.remotepartitioning.DataSourceConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.integration.annotation.ServiceActivator;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.channel.QueueChannel;
import org.springframework.integration.config.AggregatorFactoryBean;
import org.springframework.integration.config.EnableIntegration;
import org.springframework.integration.core.MessagingTemplate;
import org.springframework.integration.dsl.IntegrationFlow;
import org.springframework.integration.dsl.IntegrationFlows;
import org.springframework.integration.jms.dsl.Jms;
@@ -49,24 +42,22 @@ import org.springframework.integration.jms.dsl.Jms;
*/
@Configuration
@EnableBatchProcessing
@EnableIntegration
@EnableBatchIntegration
@Import(value = {DataSourceConfiguration.class, BrokerConfiguration.class})
public class MasterConfiguration {
private static final int GRID_SIZE = 3;
private static final long RECEIVE_TIMEOUT = 600000L;
private final JobBuilderFactory jobBuilderFactory;
private final StepBuilderFactory stepBuilderFactory;
private final RemotePartitioningMasterStepBuilderFactory masterStepBuilderFactory;
public MasterConfiguration(JobBuilderFactory jobBuilderFactory,
StepBuilderFactory stepBuilderFactory) {
RemotePartitioningMasterStepBuilderFactory masterStepBuilderFactory) {
this.jobBuilderFactory = jobBuilderFactory;
this.stepBuilderFactory = stepBuilderFactory;
this.masterStepBuilderFactory = masterStepBuilderFactory;
}
/*
@@ -89,64 +80,31 @@ public class MasterConfiguration {
* Configure inbound flow (replies coming from workers)
*/
@Bean
public QueueChannel replies() {
return new QueueChannel();
}
@Bean
public DirectChannel inboundStaging() {
public DirectChannel replies() {
return new DirectChannel();
}
@Bean
public IntegrationFlow inboundStagingFlow(ActiveMQConnectionFactory connectionFactory) {
public IntegrationFlow inboundFlow(ActiveMQConnectionFactory connectionFactory) {
return IntegrationFlows
.from(Jms.messageDrivenChannelAdapter(connectionFactory).destination("replies"))
.channel(inboundStaging())
.channel(replies())
.get();
}
/*
* Configure master step components
* Configure the master step
*/
@Bean
public Step masterStep() {
return this.stepBuilderFactory.get("masterStep")
.partitioner("slaveStep", partitioner())
.partitionHandler(partitionHandler())
return this.masterStepBuilderFactory.get("masterStep")
.partitioner("workerStep", new BasicPartitioner())
.gridSize(GRID_SIZE)
.outputChannel(requests())
.inputChannel(replies())
.build();
}
@Bean
public Partitioner partitioner() {
return new BasicPartitioner();
}
@Bean
public PartitionHandler partitionHandler() {
MessageChannelPartitionHandler partitionHandler = new MessageChannelPartitionHandler();
partitionHandler.setStepName("slaveStep");
partitionHandler.setGridSize(GRID_SIZE);
partitionHandler.setReplyChannel(replies());
MessagingTemplate template = new MessagingTemplate();
template.setDefaultChannel(requests());
template.setReceiveTimeout(RECEIVE_TIMEOUT);
partitionHandler.setMessagingOperations(template);
return partitionHandler;
}
@Bean
@ServiceActivator(inputChannel = "inboundStaging")
public AggregatorFactoryBean partitioningMessageHandler() {
AggregatorFactoryBean aggregatorFactoryBean = new AggregatorFactoryBean();
aggregatorFactoryBean.setProcessorBean(partitionHandler());
aggregatorFactoryBean.setOutputChannel(replies());
return aggregatorFactoryBean;
}
@Bean
public Job remotePartitioningJob() {
return this.jobBuilderFactory.get("remotePartitioningJob")

49
spring-batch-samples/src/main/java/org/springframework/batch/sample/remotepartitioning/aggregating/WorkerConfiguration.java
View File

@@ -19,23 +19,18 @@ import org.apache.activemq.ActiveMQConnectionFactory;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.integration.partition.BeanFactoryStepLocator;
import org.springframework.batch.integration.partition.StepExecutionRequestHandler;
import org.springframework.batch.integration.config.annotation.EnableBatchIntegration;
import org.springframework.batch.integration.partition.RemotePartitioningWorkerStepBuilderFactory;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.batch.sample.remotepartitioning.BrokerConfiguration;
import org.springframework.batch.sample.remotepartitioning.DataSourceConfiguration;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.integration.annotation.ServiceActivator;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.config.EnableIntegration;
import org.springframework.integration.dsl.IntegrationFlow;
import org.springframework.integration.dsl.IntegrationFlows;
import org.springframework.integration.jms.dsl.Jms;
@@ -48,24 +43,15 @@ import org.springframework.integration.jms.dsl.Jms;
*/
@Configuration
@EnableBatchProcessing
@EnableIntegration
@EnableBatchIntegration
@Import(value = {DataSourceConfiguration.class, BrokerConfiguration.class})
public class WorkerConfiguration {
private final StepBuilderFactory stepBuilderFactory;
private final ApplicationContext applicationContext;
private final JobExplorer jobExplorer;
private final RemotePartitioningWorkerStepBuilderFactory workerStepBuilderFactory;
public WorkerConfiguration(StepBuilderFactory stepBuilderFactory,
JobExplorer jobExplorer,
ApplicationContext applicationContext) {
this.stepBuilderFactory = stepBuilderFactory;
this.applicationContext = applicationContext;
this.jobExplorer = jobExplorer;
public WorkerConfiguration(RemotePartitioningWorkerStepBuilderFactory workerStepBuilderFactory) {
this.workerStepBuilderFactory = workerStepBuilderFactory;
}
/*
@@ -101,29 +87,20 @@ public class WorkerConfiguration {
}
/*
* Configure worker components
* Configure the worker step
*/
@Bean
@ServiceActivator(inputChannel = "requests", outputChannel = "replies")
public StepExecutionRequestHandler stepExecutionRequestHandler() {
StepExecutionRequestHandler stepExecutionRequestHandler = new StepExecutionRequestHandler();
stepExecutionRequestHandler.setJobExplorer(this.jobExplorer);
BeanFactoryStepLocator stepLocator = new BeanFactoryStepLocator();
stepLocator.setBeanFactory(this.applicationContext);
stepExecutionRequestHandler.setStepLocator(stepLocator);
return stepExecutionRequestHandler;
}
@Bean
public Step slaveStep() {
return this.stepBuilderFactory.get("slaveStep")
.tasklet(getTasklet(null))
public Step workerStep() {
return this.workerStepBuilderFactory.get("workerStep")
.inputChannel(requests())
.outputChannel(replies())
.tasklet(tasklet(null))
.build();
}
@Bean
@StepScope
public Tasklet getTasklet(@Value("#{stepExecutionContext['partition']}") String partition) {
public Tasklet tasklet(@Value("#{stepExecutionContext['partition']}") String partition) {
return (contribution, chunkContext) -> {
System.out.println("processing " + partition);
return RepeatStatus.FINISHED;

51
spring-batch-samples/src/main/java/org/springframework/batch/sample/remotepartitioning/polling/MasterConfiguration.java
View File

@@ -15,18 +15,14 @@
*/
package org.springframework.batch.sample.remotepartitioning.polling;
import javax.sql.DataSource;
import org.apache.activemq.ActiveMQConnectionFactory;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.partition.PartitionHandler;
import org.springframework.batch.core.partition.support.Partitioner;
import org.springframework.batch.integration.partition.MessageChannelPartitionHandler;
import org.springframework.batch.integration.config.annotation.EnableBatchIntegration;
import org.springframework.batch.integration.partition.RemotePartitioningMasterStepBuilderFactory;
import org.springframework.batch.sample.remotepartitioning.BasicPartitioner;
import org.springframework.batch.sample.remotepartitioning.BrokerConfiguration;
import org.springframework.batch.sample.remotepartitioning.DataSourceConfiguration;
@@ -34,8 +30,6 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.config.EnableIntegration;
import org.springframework.integration.core.MessagingTemplate;
import org.springframework.integration.dsl.IntegrationFlow;
import org.springframework.integration.dsl.IntegrationFlows;
import org.springframework.integration.jms.dsl.Jms;
@@ -48,26 +42,22 @@ import org.springframework.integration.jms.dsl.Jms;
*/
@Configuration
@EnableBatchProcessing
@EnableIntegration
@EnableBatchIntegration
@Import(value = {DataSourceConfiguration.class, BrokerConfiguration.class})
public class MasterConfiguration {
private static final int GRID_SIZE = 3;
private static final long RECEIVE_TIMEOUT = 600000L;
private static final long POLL_INTERVAL = 2000L;
private final JobBuilderFactory jobBuilderFactory;
private final StepBuilderFactory stepBuilderFactory;
private final RemotePartitioningMasterStepBuilderFactory masterStepBuilderFactory;
public MasterConfiguration(JobBuilderFactory jobBuilderFactory,
StepBuilderFactory stepBuilderFactory) {
RemotePartitioningMasterStepBuilderFactory masterStepBuilderFactory) {
this.jobBuilderFactory = jobBuilderFactory;
this.stepBuilderFactory = stepBuilderFactory;
this.masterStepBuilderFactory = masterStepBuilderFactory;
}
/*
@@ -87,38 +77,17 @@ public class MasterConfiguration {
}
/*
* Configure master step components
* Configure the master step
*/
@Bean
public Step masterStep() {
return this.stepBuilderFactory.get("masterStep")
.partitioner("slaveStep", partitioner())
.partitionHandler(partitionHandler(null))
return this.masterStepBuilderFactory.get("masterStep")
.partitioner("workerStep", new BasicPartitioner())
.gridSize(GRID_SIZE)
.outputChannel(requests())
.build();
}
@Bean
public Partitioner partitioner() {
return new BasicPartitioner();
}
@Bean
public PartitionHandler partitionHandler(DataSource dataSource) {
MessageChannelPartitionHandler partitionHandler = new MessageChannelPartitionHandler();
partitionHandler.setStepName("slaveStep");
partitionHandler.setGridSize(GRID_SIZE);
partitionHandler.setDataSource(dataSource);
partitionHandler.setPollInterval(POLL_INTERVAL);
MessagingTemplate template = new MessagingTemplate();
template.setDefaultChannel(requests());
template.setReceiveTimeout(RECEIVE_TIMEOUT);
partitionHandler.setMessagingOperations(template);
return partitionHandler;
}
@Bean
public Job remotePartitioningJob() {
return this.jobBuilderFactory.get("remotePartitioningJob")

57
spring-batch-samples/src/main/java/org/springframework/batch/sample/remotepartitioning/polling/WorkerConfiguration.java
View File

@@ -19,24 +19,18 @@ import org.apache.activemq.ActiveMQConnectionFactory;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.integration.partition.BeanFactoryStepLocator;
import org.springframework.batch.integration.partition.StepExecutionRequestHandler;
import org.springframework.batch.integration.config.annotation.EnableBatchIntegration;
import org.springframework.batch.integration.partition.RemotePartitioningWorkerStepBuilderFactory;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.batch.sample.remotepartitioning.BrokerConfiguration;
import org.springframework.batch.sample.remotepartitioning.DataSourceConfiguration;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.integration.annotation.ServiceActivator;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.channel.NullChannel;
import org.springframework.integration.config.EnableIntegration;
import org.springframework.integration.dsl.IntegrationFlow;
import org.springframework.integration.dsl.IntegrationFlows;
import org.springframework.integration.jms.dsl.Jms;
@@ -49,24 +43,15 @@ import org.springframework.integration.jms.dsl.Jms;
*/
@Configuration
@EnableBatchProcessing
@EnableIntegration
@EnableBatchIntegration
@Import(value = {DataSourceConfiguration.class, BrokerConfiguration.class})
public class WorkerConfiguration {
private final StepBuilderFactory stepBuilderFactory;
private final ApplicationContext applicationContext;
private final JobExplorer jobExplorer;
private final RemotePartitioningWorkerStepBuilderFactory workerStepBuilderFactory;
public WorkerConfiguration(StepBuilderFactory stepBuilderFactory,
JobExplorer jobExplorer,
ApplicationContext applicationContext) {
this.stepBuilderFactory = stepBuilderFactory;
this.applicationContext = applicationContext;
this.jobExplorer = jobExplorer;
public WorkerConfiguration(RemotePartitioningWorkerStepBuilderFactory workerStepBuilderFactory) {
this.workerStepBuilderFactory = workerStepBuilderFactory;
}
/*
@@ -86,37 +71,19 @@ public class WorkerConfiguration {
}
/*
* Configure outbound flow (replies going to the master)
* Configure the worker step
*/
@Bean
public NullChannel replies() {
return new NullChannel(); // replies are discarded (since the master is polling the job repository)
}
/*
* Configure worker components
*/
@Bean
@ServiceActivator(inputChannel = "requests", outputChannel = "replies")
public StepExecutionRequestHandler stepExecutionRequestHandler() {
StepExecutionRequestHandler stepExecutionRequestHandler = new StepExecutionRequestHandler();
stepExecutionRequestHandler.setJobExplorer(this.jobExplorer);
BeanFactoryStepLocator stepLocator = new BeanFactoryStepLocator();
stepLocator.setBeanFactory(this.applicationContext);
stepExecutionRequestHandler.setStepLocator(stepLocator);
return stepExecutionRequestHandler;
}
@Bean
public Step slaveStep() {
return this.stepBuilderFactory.get("slaveStep")
.tasklet(getTasklet(null))
public Step workerStep() {
return this.workerStepBuilderFactory.get("workerStep")
.inputChannel(requests())
.tasklet(tasklet(null))
.build();
}
@Bean
@StepScope
public Tasklet getTasklet(@Value("#{stepExecutionContext['partition']}") String partition) {
public Tasklet tasklet(@Value("#{stepExecutionContext['partition']}") String partition) {
return (contribution, chunkContext) -> {
System.out.println("processing " + partition);
return RepeatStatus.FINISHED;