Execute batch jobs with ApplicationRunner

This commit migrates JobLauncherCommandLineRunner to an
ApplicationRunner implementation. The latter allows to parse option
arguments (i.e. `--spring.something=value`) and makes it easy to ignore
arguments that are not meant to be passed to batch jobs.

Closes gh-19442
This commit is contained in:
Stephane Nicoll
2019-12-23 16:19:28 +01:00
parent 22fcb954d4
commit 68dc850a82
7 changed files with 288 additions and 219 deletions

View File

@@ -70,9 +70,9 @@ public class BatchAutoConfiguration {
@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "spring.batch.job", name = "enabled", havingValue = "true", matchIfMissing = true)
public JobLauncherCommandLineRunner jobLauncherCommandLineRunner(JobLauncher jobLauncher, JobExplorer jobExplorer,
public JobLauncherApplicationRunner jobLauncherApplicationRunner(JobLauncher jobLauncher, JobExplorer jobExplorer,
JobRepository jobRepository, BatchProperties properties) {
JobLauncherCommandLineRunner runner = new JobLauncherCommandLineRunner(jobLauncher, jobExplorer, jobRepository);
JobLauncherApplicationRunner runner = new JobLauncherApplicationRunner(jobLauncher, jobExplorer, jobRepository);
String jobNames = properties.getJob().getNames();
if (StringUtils.hasText(jobNames)) {
runner.setJobNames(jobNames);

View File

@@ -0,0 +1,248 @@
/*
* Copyright 2012-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.boot.autoconfigure.batch;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobExecutionException;
import org.springframework.batch.core.JobParameter;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.batch.core.configuration.JobRegistry;
import org.springframework.batch.core.converter.DefaultJobParametersConverter;
import org.springframework.batch.core.converter.JobParametersConverter;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.launch.JobParametersNotFoundException;
import org.springframework.batch.core.launch.NoSuchJobException;
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.repository.JobRestartException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.core.Ordered;
import org.springframework.core.log.LogMessage;
import org.springframework.util.Assert;
import org.springframework.util.PatternMatchUtils;
import org.springframework.util.StringUtils;
/**
* {@link ApplicationRunner} to {@link JobLauncher launch} Spring Batch jobs. Runs all
* jobs in the surrounding context by default. Can also be used to launch a specific job
* by providing a jobName
*
* @author Dave Syer
* @author Jean-Pierre Bergamin
* @author Mahmoud Ben Hassine
* @author Stephane Nicoll
* @since 2.3.0
*/
public class JobLauncherApplicationRunner implements ApplicationRunner, Ordered, ApplicationEventPublisherAware {
/**
* The default order for the command line runner.
*/
public static final int DEFAULT_ORDER = 0;
private static final Log logger = LogFactory.getLog(JobLauncherApplicationRunner.class);
private JobParametersConverter converter = new DefaultJobParametersConverter();
private final JobLauncher jobLauncher;
private final JobExplorer jobExplorer;
private final JobRepository jobRepository;
private JobRegistry jobRegistry;
private String jobNames;
private Collection<Job> jobs = Collections.emptySet();
private int order = DEFAULT_ORDER;
private ApplicationEventPublisher publisher;
/**
* Create a new {@link JobLauncherApplicationRunner}.
* @param jobLauncher to launch jobs
* @param jobExplorer to check the job repository for previous executions
* @param jobRepository to check if a job instance exists with the given parameters
* when running a job
*/
public JobLauncherApplicationRunner(JobLauncher jobLauncher, JobExplorer jobExplorer, JobRepository jobRepository) {
Assert.notNull(jobLauncher, "JobLauncher must not be null");
Assert.notNull(jobExplorer, "JobExplorer must not be null");
Assert.notNull(jobRepository, "JobRepository must not be null");
this.jobLauncher = jobLauncher;
this.jobExplorer = jobExplorer;
this.jobRepository = jobRepository;
}
public void setOrder(int order) {
this.order = order;
}
@Override
public int getOrder() {
return this.order;
}
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher publisher) {
this.publisher = publisher;
}
@Autowired(required = false)
public void setJobRegistry(JobRegistry jobRegistry) {
this.jobRegistry = jobRegistry;
}
public void setJobNames(String jobNames) {
this.jobNames = jobNames;
}
@Autowired(required = false)
public void setJobParametersConverter(JobParametersConverter converter) {
this.converter = converter;
}
@Autowired(required = false)
public void setJobs(Collection<Job> jobs) {
this.jobs = jobs;
}
@Override
public void run(ApplicationArguments args) throws Exception {
String[] jobArguments = args.getNonOptionArgs().toArray(new String[0]);
run(jobArguments);
}
public void run(String... args) throws JobExecutionException {
logger.info("Running default command line with: " + Arrays.asList(args));
launchJobFromProperties(StringUtils.splitArrayElementsIntoProperties(args, "="));
}
protected void launchJobFromProperties(Properties properties) throws JobExecutionException {
JobParameters jobParameters = this.converter.getJobParameters(properties);
executeLocalJobs(jobParameters);
executeRegisteredJobs(jobParameters);
}
private void executeLocalJobs(JobParameters jobParameters) throws JobExecutionException {
for (Job job : this.jobs) {
if (StringUtils.hasText(this.jobNames)) {
String[] jobsToRun = this.jobNames.split(",");
if (!PatternMatchUtils.simpleMatch(jobsToRun, job.getName())) {
logger.debug(LogMessage.format("Skipped job: %s", job.getName()));
continue;
}
}
execute(job, jobParameters);
}
}
private void executeRegisteredJobs(JobParameters jobParameters) throws JobExecutionException {
if (this.jobRegistry != null && StringUtils.hasText(this.jobNames)) {
String[] jobsToRun = this.jobNames.split(",");
for (String jobName : jobsToRun) {
try {
Job job = this.jobRegistry.getJob(jobName);
if (this.jobs.contains(job)) {
continue;
}
execute(job, jobParameters);
}
catch (NoSuchJobException ex) {
logger.debug(LogMessage.format("No job found in registry for job name: %s", jobName));
}
}
}
}
protected void execute(Job job, JobParameters jobParameters)
throws JobExecutionAlreadyRunningException, JobRestartException, JobInstanceAlreadyCompleteException,
JobParametersInvalidException, JobParametersNotFoundException {
JobParameters parameters = getNextJobParameters(job, jobParameters);
JobExecution execution = this.jobLauncher.run(job, parameters);
if (this.publisher != null) {
this.publisher.publishEvent(new JobExecutionEvent(execution));
}
}
private JobParameters getNextJobParameters(Job job, JobParameters jobParameters) {
if (this.jobRepository != null && this.jobRepository.isJobInstanceExists(job.getName(), jobParameters)) {
return getNextJobParametersForExisting(job, jobParameters);
}
if (job.getJobParametersIncrementer() == null) {
return jobParameters;
}
JobParameters nextParameters = new JobParametersBuilder(jobParameters, this.jobExplorer)
.getNextJobParameters(job).toJobParameters();
return merge(nextParameters, jobParameters);
}
private JobParameters getNextJobParametersForExisting(Job job, JobParameters jobParameters) {
JobExecution lastExecution = this.jobRepository.getLastJobExecution(job.getName(), jobParameters);
if (isStoppedOrFailed(lastExecution) && job.isRestartable()) {
JobParameters previousIdentifyingParameters = getGetIdentifying(lastExecution.getJobParameters());
return merge(previousIdentifyingParameters, jobParameters);
}
return jobParameters;
}
private boolean isStoppedOrFailed(JobExecution execution) {
BatchStatus status = (execution != null) ? execution.getStatus() : null;
return (status == BatchStatus.STOPPED || status == BatchStatus.FAILED);
}
private JobParameters getGetIdentifying(JobParameters parameters) {
HashMap<String, JobParameter> nonIdentifying = new LinkedHashMap<>(parameters.getParameters().size());
parameters.getParameters().forEach((key, value) -> {
if (value.isIdentifying()) {
nonIdentifying.put(key, value);
}
});
return new JobParameters(nonIdentifying);
}
private JobParameters merge(JobParameters parameters, JobParameters additionals) {
Map<String, JobParameter> merged = new LinkedHashMap<>();
merged.putAll(parameters.getParameters());
merged.putAll(additionals.getParameters());
return new JobParameters(merged);
}
}

View File

@@ -16,48 +16,13 @@
package org.springframework.boot.autoconfigure.batch;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobExecutionException;
import org.springframework.batch.core.JobParameter;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.batch.core.configuration.JobRegistry;
import org.springframework.batch.core.converter.DefaultJobParametersConverter;
import org.springframework.batch.core.converter.JobParametersConverter;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.launch.JobParametersNotFoundException;
import org.springframework.batch.core.launch.NoSuchJobException;
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.repository.JobRestartException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.core.Ordered;
import org.springframework.core.log.LogMessage;
import org.springframework.util.Assert;
import org.springframework.util.PatternMatchUtils;
import org.springframework.util.StringUtils;
import org.springframework.boot.ApplicationRunner;
/**
* {@link CommandLineRunner} to {@link JobLauncher launch} Spring Batch jobs. Runs all
* {@link ApplicationRunner} to {@link JobLauncher launch} Spring Batch jobs. Runs all
* jobs in the surrounding context by default. Can also be used to launch a specific job
* by providing a jobName
*
@@ -65,33 +30,10 @@ import org.springframework.util.StringUtils;
* @author Jean-Pierre Bergamin
* @author Mahmoud Ben Hassine
* @since 1.0.0
* @deprecated since 2.2.0 in favor of {@link JobLauncherApplicationRunner}
*/
public class JobLauncherCommandLineRunner implements CommandLineRunner, Ordered, ApplicationEventPublisherAware {
/**
* The default order for the command line runner.
*/
public static final int DEFAULT_ORDER = 0;
private static final Log logger = LogFactory.getLog(JobLauncherCommandLineRunner.class);
private JobParametersConverter converter = new DefaultJobParametersConverter();
private final JobLauncher jobLauncher;
private final JobExplorer jobExplorer;
private final JobRepository jobRepository;
private JobRegistry jobRegistry;
private String jobNames;
private Collection<Job> jobs = Collections.emptySet();
private int order = DEFAULT_ORDER;
private ApplicationEventPublisher publisher;
@Deprecated
public class JobLauncherCommandLineRunner extends JobLauncherApplicationRunner {
/**
* Create a new {@link JobLauncherCommandLineRunner}.
@@ -101,141 +43,7 @@ public class JobLauncherCommandLineRunner implements CommandLineRunner, Ordered,
* when running a job
*/
public JobLauncherCommandLineRunner(JobLauncher jobLauncher, JobExplorer jobExplorer, JobRepository jobRepository) {
Assert.notNull(jobLauncher, "JobLauncher must not be null");
Assert.notNull(jobExplorer, "JobExplorer must not be null");
Assert.notNull(jobRepository, "JobRepository must not be null");
this.jobLauncher = jobLauncher;
this.jobExplorer = jobExplorer;
this.jobRepository = jobRepository;
}
public void setOrder(int order) {
this.order = order;
}
@Override
public int getOrder() {
return this.order;
}
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher publisher) {
this.publisher = publisher;
}
@Autowired(required = false)
public void setJobRegistry(JobRegistry jobRegistry) {
this.jobRegistry = jobRegistry;
}
public void setJobNames(String jobNames) {
this.jobNames = jobNames;
}
@Autowired(required = false)
public void setJobParametersConverter(JobParametersConverter converter) {
this.converter = converter;
}
@Autowired(required = false)
public void setJobs(Collection<Job> jobs) {
this.jobs = jobs;
}
@Override
public void run(String... args) throws JobExecutionException {
logger.info("Running default command line with: " + Arrays.asList(args));
launchJobFromProperties(StringUtils.splitArrayElementsIntoProperties(args, "="));
}
protected void launchJobFromProperties(Properties properties) throws JobExecutionException {
JobParameters jobParameters = this.converter.getJobParameters(properties);
executeLocalJobs(jobParameters);
executeRegisteredJobs(jobParameters);
}
private void executeLocalJobs(JobParameters jobParameters) throws JobExecutionException {
for (Job job : this.jobs) {
if (StringUtils.hasText(this.jobNames)) {
String[] jobsToRun = this.jobNames.split(",");
if (!PatternMatchUtils.simpleMatch(jobsToRun, job.getName())) {
logger.debug(LogMessage.format("Skipped job: %s", job.getName()));
continue;
}
}
execute(job, jobParameters);
}
}
private void executeRegisteredJobs(JobParameters jobParameters) throws JobExecutionException {
if (this.jobRegistry != null && StringUtils.hasText(this.jobNames)) {
String[] jobsToRun = this.jobNames.split(",");
for (String jobName : jobsToRun) {
try {
Job job = this.jobRegistry.getJob(jobName);
if (this.jobs.contains(job)) {
continue;
}
execute(job, jobParameters);
}
catch (NoSuchJobException ex) {
logger.debug(LogMessage.format("No job found in registry for job name: %s", jobName));
}
}
}
}
protected void execute(Job job, JobParameters jobParameters)
throws JobExecutionAlreadyRunningException, JobRestartException, JobInstanceAlreadyCompleteException,
JobParametersInvalidException, JobParametersNotFoundException {
JobParameters parameters = getNextJobParameters(job, jobParameters);
JobExecution execution = this.jobLauncher.run(job, parameters);
if (this.publisher != null) {
this.publisher.publishEvent(new JobExecutionEvent(execution));
}
}
private JobParameters getNextJobParameters(Job job, JobParameters jobParameters) {
if (this.jobRepository != null && this.jobRepository.isJobInstanceExists(job.getName(), jobParameters)) {
return getNextJobParametersForExisting(job, jobParameters);
}
if (job.getJobParametersIncrementer() == null) {
return jobParameters;
}
JobParameters nextParameters = new JobParametersBuilder(jobParameters, this.jobExplorer)
.getNextJobParameters(job).toJobParameters();
return merge(nextParameters, jobParameters);
}
private JobParameters getNextJobParametersForExisting(Job job, JobParameters jobParameters) {
JobExecution lastExecution = this.jobRepository.getLastJobExecution(job.getName(), jobParameters);
if (isStoppedOrFailed(lastExecution) && job.isRestartable()) {
JobParameters previousIdentifyingParameters = getGetIdentifying(lastExecution.getJobParameters());
return merge(previousIdentifyingParameters, jobParameters);
}
return jobParameters;
}
private boolean isStoppedOrFailed(JobExecution execution) {
BatchStatus status = (execution != null) ? execution.getStatus() : null;
return (status == BatchStatus.STOPPED || status == BatchStatus.FAILED);
}
private JobParameters getGetIdentifying(JobParameters parameters) {
HashMap<String, JobParameter> nonIdentifying = new LinkedHashMap<>(parameters.getParameters().size());
parameters.getParameters().forEach((key, value) -> {
if (value.isIdentifying()) {
nonIdentifying.put(key, value);
}
});
return new JobParameters(nonIdentifying);
}
private JobParameters merge(JobParameters parameters, JobParameters additionals) {
Map<String, JobParameter> merged = new LinkedHashMap<>();
merged.putAll(parameters.getParameters());
merged.putAll(additionals.getParameters());
return new JobParameters(merged);
super(jobLauncher, jobExplorer, jobRepository);
}
}

View File

@@ -28,6 +28,7 @@ import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.JobRegistry;
import org.springframework.batch.core.configuration.annotation.BatchConfigurer;
@@ -43,6 +44,7 @@ import org.springframework.batch.core.repository.support.MapJobRepositoryFactory
import org.springframework.batch.support.transaction.ResourcelessTransactionManager;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.DefaultApplicationArguments;
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.autoconfigure.TestAutoConfigurationPackage;
import org.springframework.boot.autoconfigure.jdbc.EmbeddedDataSourceConfiguration;
@@ -126,8 +128,25 @@ class BatchAutoConfigurationTests {
this.contextRunner.withUserConfiguration(JobConfiguration.class, EmbeddedDataSourceConfiguration.class)
.run((context) -> {
assertThat(context).hasSingleBean(JobLauncher.class);
context.getBean(JobLauncherCommandLineRunner.class).run();
assertThat(context.getBean(JobRepository.class).getLastJobExecution("job", new JobParameters()))
context.getBean(JobLauncherApplicationRunner.class)
.run(new DefaultApplicationArguments("jobParam=test"));
JobParameters jobParameters = new JobParametersBuilder().addString("jobParam", "test")
.toJobParameters();
assertThat(context.getBean(JobRepository.class).getLastJobExecution("job", jobParameters))
.isNotNull();
});
}
@Test
void testDefinesAndLaunchesJobIgnoreOptionArguments() {
this.contextRunner.withUserConfiguration(JobConfiguration.class, EmbeddedDataSourceConfiguration.class)
.run((context) -> {
assertThat(context).hasSingleBean(JobLauncher.class);
context.getBean(JobLauncherApplicationRunner.class)
.run(new DefaultApplicationArguments("--spring.property=value", "jobParam=test"));
JobParameters jobParameters = new JobParametersBuilder().addString("jobParam", "test")
.toJobParameters();
assertThat(context.getBean(JobRepository.class).getLastJobExecution("job", jobParameters))
.isNotNull();
});
}
@@ -139,7 +158,7 @@ class BatchAutoConfigurationTests {
EmbeddedDataSourceConfiguration.class)
.withPropertyValues("spring.batch.job.names:discreteRegisteredJob").run((context) -> {
assertThat(context).hasSingleBean(JobLauncher.class);
context.getBean(JobLauncherCommandLineRunner.class).run();
context.getBean(JobLauncherApplicationRunner.class).run();
assertThat(context.getBean(JobRepository.class).getLastJobExecution("discreteRegisteredJob",
new JobParameters())).isNotNull();
});
@@ -151,7 +170,7 @@ class BatchAutoConfigurationTests {
.withUserConfiguration(NamedJobConfigurationWithLocalJob.class, EmbeddedDataSourceConfiguration.class)
.withPropertyValues("spring.batch.job.names:discreteLocalJob").run((context) -> {
assertThat(context).hasSingleBean(JobLauncher.class);
context.getBean(JobLauncherCommandLineRunner.class).run();
context.getBean(JobLauncherApplicationRunner.class).run();
assertThat(context.getBean(JobRepository.class).getLastJobExecution("discreteLocalJob",
new JobParameters())).isNotNull();
});

View File

@@ -49,7 +49,7 @@ class BatchAutoConfigurationWithoutJdbcTests {
@Test
void whenThereIsNoJdbcOnTheClasspathThenComponentsAreStillAutoConfigured() {
this.contextRunner.run((context) -> {
assertThat(context).hasSingleBean(JobLauncherCommandLineRunner.class);
assertThat(context).hasSingleBean(JobLauncherApplicationRunner.class);
assertThat(context).hasSingleBean(JobExecutionExitCodeGenerator.class);
assertThat(context).hasSingleBean(SimpleJobOperator.class);
});

View File

@@ -50,17 +50,18 @@ import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.assertj.core.api.Assertions.fail;
/**
* Tests for {@link JobLauncherCommandLineRunner}.
* Tests for {@link JobLauncherApplicationRunner}.
*
* @author Dave Syer
* @author Jean-Pierre Bergamin
* @author Mahmoud Ben Hassine
* @author Stephane Nicoll
*/
class JobLauncherCommandLineRunnerTests {
class JobLauncherApplicationRunnerTests {
private AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext();
private JobLauncherCommandLineRunner runner;
private JobLauncherApplicationRunner runner;
private JobExplorer jobExplorer;
@@ -85,7 +86,7 @@ class JobLauncherCommandLineRunnerTests {
this.step = this.steps.get("step").tasklet(tasklet).build();
this.job = this.jobs.get("job").start(this.step).build();
this.jobExplorer = this.context.getBean(JobExplorer.class);
this.runner = new JobLauncherCommandLineRunner(jobLauncher, this.jobExplorer, jobRepository);
this.runner = new JobLauncherApplicationRunner(jobLauncher, this.jobExplorer, jobRepository);
this.context.getBean(BatchConfiguration.class).clear();
}