From 93156fdabcc7a49dfd1df42225caa4fa75b7245c Mon Sep 17 00:00:00 2001 From: Michael Minella Date: Mon, 21 Dec 2020 00:38:53 -0600 Subject: [PATCH] Added documentation for the Single Step Batch Job Starter This commit does two things: 1. Adds documentation to the reference documentation for the single step batch job starter. 2. It standardizes the naming of the properties across the various readers/writers the autoconfiguration provides. Polished docs on merge --- .../FlatFileItemReaderAutoConfiguration.java | 2 +- .../FlatFileItemReaderProperties.java | 2 +- .../FlatFileItemWriterAutoConfiguration.java | 2 +- .../FlatFileItemWriterProperties.java | 2 +- ...JdbcBatchItemWriterAutoConfiguration.java} | 16 +- ...ava => JdbcBatchItemWriterProperties.java} | 6 +- ...JdbcCursorItemReaderAutoConfiguration.java | 2 +- .../jdbc/JdbcCursorItemReaderProperties.java | 2 +- .../main/resources/META-INF/spring.factories | 2 +- ...tFileItemReaderAutoConfigurationTests.java | 76 +-- ...tFileItemWriterAutoConfigurationTests.java | 74 +-- ...atchItemWriterAutoConfigurationTests.java} | 18 +- ...ursorItemReaderAutoConfigurationTests.java | 34 +- .../src/main/asciidoc/batch-starter.adoc | 532 ++++++++++++++++++ .../src/main/asciidoc/index.adoc | 2 + .../resources/application-ffreader.properties | 18 +- .../resources/application-ffwriter.properties | 22 +- .../application-jdbcreader.properties | 4 +- .../application-jdbcwriter.properties | 4 +- 19 files changed, 677 insertions(+), 143 deletions(-) rename spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/{JdbcItemWriterAutoConfiguration.java => JdbcBatchItemWriterAutoConfiguration.java} (84%) rename spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/{JdbcItemWriterProperties.java => JdbcBatchItemWriterProperties.java} (92%) rename spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/{JdbcItemWriterAutoConfigurationTests.java => JdbcBatchItemWriterAutoConfigurationTests.java} (94%) create mode 100644 spring-cloud-task-docs/src/main/asciidoc/batch-starter.adoc diff --git a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemReaderAutoConfiguration.java b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemReaderAutoConfiguration.java index d2a3a4b7..31ba723b 100644 --- a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemReaderAutoConfiguration.java +++ b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemReaderAutoConfiguration.java @@ -71,7 +71,7 @@ public class FlatFileItemReaderAutoConfiguration { @Bean @ConditionalOnMissingBean - @ConditionalOnProperty(prefix = "spring.batch.job.flatfilereader", name = "name") + @ConditionalOnProperty(prefix = "spring.batch.job.flatfileitemreader", name = "name") public FlatFileItemReader> itemReader() { FlatFileItemReaderBuilder> mapFlatFileItemReaderBuilder = new FlatFileItemReaderBuilder>() .name(this.properties.getName()).resource(this.properties.getResource()) diff --git a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemReaderProperties.java b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemReaderProperties.java index 7cab1484..c43dab14 100644 --- a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemReaderProperties.java +++ b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemReaderProperties.java @@ -31,7 +31,7 @@ import org.springframework.core.io.Resource; * @author Michael Minella * @since 2.3 */ -@ConfigurationProperties(prefix = "spring.batch.job.flatfilereader") +@ConfigurationProperties(prefix = "spring.batch.job.flatfileitemreader") public class FlatFileItemReaderProperties { private boolean saveState = true; diff --git a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemWriterAutoConfiguration.java b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemWriterAutoConfiguration.java index ea135067..b6734e76 100644 --- a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemWriterAutoConfiguration.java +++ b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemWriterAutoConfiguration.java @@ -66,7 +66,7 @@ public class FlatFileItemWriterAutoConfiguration { @Bean @ConditionalOnMissingBean - @ConditionalOnProperty(prefix = "spring.batch.job.flatfilewriter", name = "name") + @ConditionalOnProperty(prefix = "spring.batch.job.flatfileitemwriter", name = "name") public FlatFileItemWriter> itemWriter() { if (this.properties.isDelimited() && this.properties.isFormatted()) { diff --git a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemWriterProperties.java b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemWriterProperties.java index cdce55da..1776f38c 100644 --- a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemWriterProperties.java +++ b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemWriterProperties.java @@ -28,7 +28,7 @@ import org.springframework.core.io.Resource; * @author Michael Minella * @since 2.3 */ -@ConfigurationProperties(prefix = "spring.batch.job.flatfilewriter") +@ConfigurationProperties(prefix = "spring.batch.job.flatfileitemwriter") public class FlatFileItemWriterProperties { private Resource resource; diff --git a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcItemWriterAutoConfiguration.java b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcBatchItemWriterAutoConfiguration.java similarity index 84% rename from spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcItemWriterAutoConfiguration.java rename to spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcBatchItemWriterAutoConfiguration.java index d1e88edf..1d6a14c3 100644 --- a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcItemWriterAutoConfiguration.java +++ b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcBatchItemWriterAutoConfiguration.java @@ -20,9 +20,9 @@ import java.util.Map; import javax.sql.DataSource; -import org.springframework.batch.item.ItemWriter; import org.springframework.batch.item.database.ItemPreparedStatementSetter; import org.springframework.batch.item.database.ItemSqlParameterSourceProvider; +import org.springframework.batch.item.database.JdbcBatchItemWriter; import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.AutoConfigureAfter; @@ -41,9 +41,9 @@ import org.springframework.context.annotation.Configuration; * @since 2.3 */ @Configuration -@EnableConfigurationProperties(JdbcItemWriterProperties.class) +@EnableConfigurationProperties(JdbcBatchItemWriterProperties.class) @AutoConfigureAfter(BatchAutoConfiguration.class) -public class JdbcItemWriterAutoConfiguration { +public class JdbcBatchItemWriterAutoConfiguration { @Autowired(required = false) private ItemPreparedStatementSetter itemPreparedStatementSetter; @@ -51,20 +51,20 @@ public class JdbcItemWriterAutoConfiguration { @Autowired(required = false) private ItemSqlParameterSourceProvider itemSqlParameterSourceProvider; - private JdbcItemWriterProperties properties; + private JdbcBatchItemWriterProperties properties; private DataSource dataSource; - public JdbcItemWriterAutoConfiguration(DataSource dataSource, - JdbcItemWriterProperties properties) { + public JdbcBatchItemWriterAutoConfiguration(DataSource dataSource, + JdbcBatchItemWriterProperties properties) { this.dataSource = dataSource; this.properties = properties; } @Bean @ConditionalOnMissingBean - @ConditionalOnProperty(prefix = "spring.batch.job.jdbcwriter", name = "name") - public ItemWriter> itemWriter() { + @ConditionalOnProperty(prefix = "spring.batch.job.jdbcbatchitemwriter", name = "name") + public JdbcBatchItemWriter> itemWriter() { JdbcBatchItemWriterBuilder> jdbcBatchItemWriterBuilder = new JdbcBatchItemWriterBuilder>() .dataSource(this.dataSource).sql(this.properties.getSql()); diff --git a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcItemWriterProperties.java b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcBatchItemWriterProperties.java similarity index 92% rename from spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcItemWriterProperties.java rename to spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcBatchItemWriterProperties.java index 7a40f206..3b36dfcc 100644 --- a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcItemWriterProperties.java +++ b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcBatchItemWriterProperties.java @@ -19,13 +19,13 @@ package org.springframework.cloud.task.batch.autoconfigure.jdbc; import org.springframework.boot.context.properties.ConfigurationProperties; /** - * Properties to configure a {@code JdbcItemWriter}. + * Properties to configure a {@code JdbcBatchItemWriter}. * * @author Glenn Renfro * @since 2.3 */ -@ConfigurationProperties(prefix = "spring.batch.job.jdbcwriter") -public class JdbcItemWriterProperties { +@ConfigurationProperties(prefix = "spring.batch.job.jdbcbatchitemwriter") +public class JdbcBatchItemWriterProperties { private String name; diff --git a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcCursorItemReaderAutoConfiguration.java b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcCursorItemReaderAutoConfiguration.java index 66a7f028..6a8dfeab 100644 --- a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcCursorItemReaderAutoConfiguration.java +++ b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcCursorItemReaderAutoConfiguration.java @@ -43,7 +43,7 @@ import org.springframework.jdbc.core.RowMapper; @Configuration @EnableConfigurationProperties(JdbcCursorItemReaderProperties.class) @AutoConfigureAfter(BatchAutoConfiguration.class) -@ConditionalOnProperty(prefix = "spring.batch.job.jdbccursorreader", name = "name") +@ConditionalOnProperty(prefix = "spring.batch.job.jdbccursoritemreader", name = "name") public class JdbcCursorItemReaderAutoConfiguration { private final JdbcCursorItemReaderProperties properties; diff --git a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcCursorItemReaderProperties.java b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcCursorItemReaderProperties.java index 38e40bab..d447783b 100644 --- a/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcCursorItemReaderProperties.java +++ b/spring-cloud-starter-single-step-batch-job/src/main/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcCursorItemReaderProperties.java @@ -22,7 +22,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties; * @author Michael Minella * @since 2.3 */ -@ConfigurationProperties(prefix = "spring.batch.job.jdbccursorreader") +@ConfigurationProperties(prefix = "spring.batch.job.jdbccursoritemreader") public class JdbcCursorItemReaderProperties { private boolean saveState = true; diff --git a/spring-cloud-starter-single-step-batch-job/src/main/resources/META-INF/spring.factories b/spring-cloud-starter-single-step-batch-job/src/main/resources/META-INF/spring.factories index bd226b89..846202f3 100644 --- a/spring-cloud-starter-single-step-batch-job/src/main/resources/META-INF/spring.factories +++ b/spring-cloud-starter-single-step-batch-job/src/main/resources/META-INF/spring.factories @@ -2,7 +2,7 @@ org.springframework.boot.autoconfigure.EnableAutoConfiguration=org.springframewo org.springframework.cloud.task.batch.autoconfigure.RangeConverter,\ org.springframework.cloud.task.batch.autoconfigure.SingleStepJobAutoConfiguration,\ org.springframework.cloud.task.batch.autoconfigure.flatfile.FlatFileItemWriterAutoConfiguration, \ - org.springframework.cloud.task.batch.autoconfigure.jdbc.JdbcItemWriterAutoConfiguration, \ + org.springframework.cloud.task.batch.autoconfigure.jdbc.JdbcBatchItemWriterAutoConfiguration, \ org.springframework.cloud.task.batch.autoconfigure.jdbc.JdbcCursorItemReaderAutoConfiguration, \ org.springframework.cloud.task.batch.autoconfigure.rabbit.AmqpItemReaderAutoConfiguration, \ org.springframework.cloud.task.batch.autoconfigure.rabbit.AmqpItemWriterAutoConfiguration, \ diff --git a/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemReaderAutoConfigurationTests.java b/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemReaderAutoConfigurationTests.java index 0a960411..41a8bbb4 100644 --- a/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemReaderAutoConfigurationTests.java +++ b/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemReaderAutoConfigurationTests.java @@ -85,21 +85,21 @@ public class FlatFileItemReaderAutoConfigurationTests { FlatFileItemReaderAutoConfiguration.class)) .withPropertyValues("spring.batch.job.jobName=job", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.flatfilereader.savestate=true", - "spring.batch.job.flatfilereader.name=fullDelimitedConfiguration", - "spring.batch.job.flatfilereader.maxItemCount=5", - "spring.batch.job.flatfilereader.currentItemCount=2", - "spring.batch.job.flatfilereader.comments=#,$", - "spring.batch.job.flatfilereader.resource=/testUTF16.csv", - "spring.batch.job.flatfilereader.strict=true", - "spring.batch.job.flatfilereader.encoding=UTF-16", - "spring.batch.job.flatfilereader.linesToSkip=1", - "spring.batch.job.flatfilereader.delimited=true", - "spring.batch.job.flatfilereader.delimiter=@", - "spring.batch.job.flatfilereader.quoteCharacter=%", - "spring.batch.job.flatfilereader.includedFields=1,3,5", - "spring.batch.job.flatfilereader.names=foo,bar,baz", - "spring.batch.job.flatfilereader.parsingStrict=false"); + "spring.batch.job.flatfileitemreader.savestate=true", + "spring.batch.job.flatfileitemreader.name=fullDelimitedConfiguration", + "spring.batch.job.flatfileitemreader.maxItemCount=5", + "spring.batch.job.flatfileitemreader.currentItemCount=2", + "spring.batch.job.flatfileitemreader.comments=#,$", + "spring.batch.job.flatfileitemreader.resource=/testUTF16.csv", + "spring.batch.job.flatfileitemreader.strict=true", + "spring.batch.job.flatfileitemreader.encoding=UTF-16", + "spring.batch.job.flatfileitemreader.linesToSkip=1", + "spring.batch.job.flatfileitemreader.delimited=true", + "spring.batch.job.flatfileitemreader.delimiter=@", + "spring.batch.job.flatfileitemreader.quoteCharacter=%", + "spring.batch.job.flatfileitemreader.includedFields=1,3,5", + "spring.batch.job.flatfileitemreader.names=foo,bar,baz", + "spring.batch.job.flatfileitemreader.parsingStrict=false"); applicationContextRunner.run((context) -> { JobLauncher jobLauncher = context.getBean(JobLauncher.class); @@ -142,15 +142,15 @@ public class FlatFileItemReaderAutoConfigurationTests { FlatFileItemReaderAutoConfiguration.class, RangeConverter.class)) .withPropertyValues("spring.batch.job.jobName=job", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.flatfilereader.savestate=true", - "spring.batch.job.flatfilereader.name=fixedWidthConfiguration", - "spring.batch.job.flatfilereader.comments=#,$", - "spring.batch.job.flatfilereader.resource=/test.txt", - "spring.batch.job.flatfilereader.strict=true", - "spring.batch.job.flatfilereader.fixedLength=true", - "spring.batch.job.flatfilereader.ranges=3-4,7-8,11", - "spring.batch.job.flatfilereader.names=foo,bar,baz", - "spring.batch.job.flatfilereader.parsingStrict=false"); + "spring.batch.job.flatfileitemreader.savestate=true", + "spring.batch.job.flatfileitemreader.name=fixedWidthConfiguration", + "spring.batch.job.flatfileitemreader.comments=#,$", + "spring.batch.job.flatfileitemreader.resource=/test.txt", + "spring.batch.job.flatfileitemreader.strict=true", + "spring.batch.job.flatfileitemreader.fixedLength=true", + "spring.batch.job.flatfileitemreader.ranges=3-4,7-8,11", + "spring.batch.job.flatfileitemreader.names=foo,bar,baz", + "spring.batch.job.flatfileitemreader.parsingStrict=false"); applicationContextRunner.run((context) -> { JobLauncher jobLauncher = context.getBean(JobLauncher.class); @@ -202,9 +202,9 @@ public class FlatFileItemReaderAutoConfigurationTests { FlatFileItemReaderAutoConfiguration.class)) .withPropertyValues("spring.batch.job.jobName=job", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.flatfilereader.name=fixedWidthConfiguration", - "spring.batch.job.flatfilereader.resource=/test.txt", - "spring.batch.job.flatfilereader.strict=true"); + "spring.batch.job.flatfileitemreader.name=fixedWidthConfiguration", + "spring.batch.job.flatfileitemreader.resource=/test.txt", + "spring.batch.job.flatfileitemreader.strict=true"); applicationContextRunner.run((context) -> { JobLauncher jobLauncher = context.getBean(JobLauncher.class); @@ -242,13 +242,13 @@ public class FlatFileItemReaderAutoConfigurationTests { FlatFileItemReaderAutoConfiguration.class, RangeConverter.class)) .withPropertyValues("spring.batch.job.jobName=job", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.flatfilereader.name=fixedWidthConfiguration", - "spring.batch.job.flatfilereader.resource=/test.txt", - "spring.batch.job.flatfilereader.linesToSkip=2", - "spring.batch.job.flatfilereader.fixedLength=true", - "spring.batch.job.flatfilereader.ranges=3-4,7-8,11", - "spring.batch.job.flatfilereader.names=foo,bar,baz", - "spring.batch.job.flatfilereader.strict=true"); + "spring.batch.job.flatfileitemreader.name=fixedWidthConfiguration", + "spring.batch.job.flatfileitemreader.resource=/test.txt", + "spring.batch.job.flatfileitemreader.linesToSkip=2", + "spring.batch.job.flatfileitemreader.fixedLength=true", + "spring.batch.job.flatfileitemreader.ranges=3-4,7-8,11", + "spring.batch.job.flatfileitemreader.names=foo,bar,baz", + "spring.batch.job.flatfileitemreader.strict=true"); applicationContextRunner.run((context) -> { JobLauncher jobLauncher = context.getBean(JobLauncher.class); @@ -287,10 +287,10 @@ public class FlatFileItemReaderAutoConfigurationTests { FlatFileItemReaderAutoConfiguration.class, RangeConverter.class)) .withPropertyValues("spring.batch.job.jobName=job", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.flatfilereader.name=fixedWidthConfiguration", - "spring.batch.job.flatfilereader.resource=/test.txt", - "spring.batch.job.flatfilereader.maxItemCount=1", - "spring.batch.job.flatfilereader.strict=true"); + "spring.batch.job.flatfileitemreader.name=fixedWidthConfiguration", + "spring.batch.job.flatfileitemreader.resource=/test.txt", + "spring.batch.job.flatfileitemreader.maxItemCount=1", + "spring.batch.job.flatfileitemreader.strict=true"); applicationContextRunner.run((context) -> { JobLauncher jobLauncher = context.getBean(JobLauncher.class); diff --git a/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemWriterAutoConfigurationTests.java b/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemWriterAutoConfigurationTests.java index 087dcd65..f92bfd76 100644 --- a/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemWriterAutoConfigurationTests.java +++ b/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/flatfile/FlatFileItemWriterAutoConfigurationTests.java @@ -144,19 +144,19 @@ public class FlatFileItemWriterAutoConfigurationTests { FlatFileItemWriterAutoConfiguration.class)) .withPropertyValues("spring.batch.job.jobName=job", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.flatfilewriter.name=fooWriter", + "spring.batch.job.flatfileitemwriter.name=fooWriter", String.format( - "spring.batch.job.flatfilewriter.resource=file://%s", + "spring.batch.job.flatfileitemwriter.resource=file://%s", this.outputFile.getAbsolutePath()), - "spring.batch.job.flatfilewriter.encoding=UTF-16", - "spring.batch.job.flatfilewriter.saveState=false", - "spring.batch.job.flatfilewriter.shouldDeleteIfEmpty=true", - "spring.batch.job.flatfilewriter.delimited=true", - "spring.batch.job.flatfilewriter.names=item", - "spring.batch.job.flatfilewriter.append=true", - "spring.batch.job.flatfilewriter.forceSync=true", - "spring.batch.job.flatfilewriter.shouldDeleteIfExists=false", - "spring.batch.job.flatfilewriter.transactional=false"); + "spring.batch.job.flatfileitemwriter.encoding=UTF-16", + "spring.batch.job.flatfileitemwriter.saveState=false", + "spring.batch.job.flatfileitemwriter.shouldDeleteIfEmpty=true", + "spring.batch.job.flatfileitemwriter.delimited=true", + "spring.batch.job.flatfileitemwriter.names=item", + "spring.batch.job.flatfileitemwriter.append=true", + "spring.batch.job.flatfileitemwriter.forceSync=true", + "spring.batch.job.flatfileitemwriter.shouldDeleteIfExists=false", + "spring.batch.job.flatfileitemwriter.transactional=false"); applicationContextRunner.run((context) -> { JobLauncher jobLauncher = context.getBean(JobLauncher.class); @@ -200,16 +200,16 @@ public class FlatFileItemWriterAutoConfigurationTests { FlatFileItemWriterAutoConfiguration.class)) .withPropertyValues("spring.batch.job.jobName=job", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=2", - "spring.batch.job.flatfilewriter.name=fooWriter", + "spring.batch.job.flatfileitemwriter.name=fooWriter", String.format( - "spring.batch.job.flatfilewriter.resource=file://%s", + "spring.batch.job.flatfileitemwriter.resource=file://%s", this.outputFile.getAbsolutePath()), - "spring.batch.job.flatfilewriter.encoding=UTF-8", - "spring.batch.job.flatfilewriter.formatted=true", - "spring.batch.job.flatfilewriter.names=item", - "spring.batch.job.flatfilewriter.format=item = %s", - "spring.batch.job.flatfilewriter.minimumLength=8", - "spring.batch.job.flatfilewriter.maximumLength=10"); + "spring.batch.job.flatfileitemwriter.encoding=UTF-8", + "spring.batch.job.flatfileitemwriter.formatted=true", + "spring.batch.job.flatfileitemwriter.names=item", + "spring.batch.job.flatfileitemwriter.format=item = %s", + "spring.batch.job.flatfileitemwriter.minimumLength=8", + "spring.batch.job.flatfileitemwriter.maximumLength=10"); applicationContextRunner.run((context) -> { JobLauncher jobLauncher = context.getBean(JobLauncher.class); @@ -243,14 +243,14 @@ public class FlatFileItemWriterAutoConfigurationTests { FlatFileItemWriterAutoConfiguration.class)) .withPropertyValues("spring.batch.job.jobName=job", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.flatfilewriter.name=fooWriter", + "spring.batch.job.flatfileitemwriter.name=fooWriter", String.format( - "spring.batch.job.flatfilewriter.resource=file://%s", + "spring.batch.job.flatfileitemwriter.resource=file://%s", this.outputFile.getAbsolutePath()), - "spring.batch.job.flatfilewriter.encoding=UTF-8", - "spring.batch.job.flatfilewriter.formatted=true", - "spring.batch.job.flatfilewriter.names=item", - "spring.batch.job.flatfilewriter.format=item = %s"); + "spring.batch.job.flatfileitemwriter.encoding=UTF-8", + "spring.batch.job.flatfileitemwriter.formatted=true", + "spring.batch.job.flatfileitemwriter.names=item", + "spring.batch.job.flatfileitemwriter.format=item = %s"); applicationContextRunner.run((context) -> { JobLauncher jobLauncher = context.getBean(JobLauncher.class); @@ -284,12 +284,12 @@ public class FlatFileItemWriterAutoConfigurationTests { FlatFileItemWriterAutoConfiguration.class)) .withPropertyValues("spring.batch.job.jobName=job", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.flatfilewriter.name=fooWriter", + "spring.batch.job.flatfileitemwriter.name=fooWriter", String.format( - "spring.batch.job.flatfilewriter.resource=file://%s", + "spring.batch.job.flatfileitemwriter.resource=file://%s", this.outputFile.getAbsolutePath()), - "spring.batch.job.flatfilewriter.encoding=UTF-8", - "spring.batch.job.flatfilewriter.delimited=true"); + "spring.batch.job.flatfileitemwriter.encoding=UTF-8", + "spring.batch.job.flatfileitemwriter.delimited=true"); applicationContextRunner.run((context) -> { JobLauncher jobLauncher = context.getBean(JobLauncher.class); @@ -323,11 +323,11 @@ public class FlatFileItemWriterAutoConfigurationTests { FlatFileItemWriterAutoConfiguration.class)) .withPropertyValues("spring.batch.job.jobName=job", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.flatfilewriter.name=fooWriter", + "spring.batch.job.flatfileitemwriter.name=fooWriter", String.format( - "spring.batch.job.flatfilewriter.resource=file://%s", + "spring.batch.job.flatfileitemwriter.resource=file://%s", this.outputFile.getAbsolutePath()), - "spring.batch.job.flatfilewriter.encoding=UTF-8"); + "spring.batch.job.flatfileitemwriter.encoding=UTF-8"); applicationContextRunner.run((context) -> { JobLauncher jobLauncher = context.getBean(JobLauncher.class); @@ -361,13 +361,13 @@ public class FlatFileItemWriterAutoConfigurationTests { FlatFileItemWriterAutoConfiguration.class)) .withPropertyValues("spring.batch.job.jobName=job", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.flatfilewriter.name=fooWriter", + "spring.batch.job.flatfileitemwriter.name=fooWriter", String.format( - "spring.batch.job.flatfilewriter.resource=file://%s", + "spring.batch.job.flatfileitemwriter.resource=file://%s", this.outputFile.getAbsolutePath()), - "spring.batch.job.flatfilewriter.encoding=UTF-8", - "spring.batch.job.flatfilewriter.delimited=true", - "spring.batch.job.flatfilewriter.names=item"); + "spring.batch.job.flatfileitemwriter.encoding=UTF-8", + "spring.batch.job.flatfileitemwriter.delimited=true", + "spring.batch.job.flatfileitemwriter.names=item"); applicationContextRunner.run((context) -> { JobLauncher jobLauncher = context.getBean(JobLauncher.class); diff --git a/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcItemWriterAutoConfigurationTests.java b/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcBatchItemWriterAutoConfigurationTests.java similarity index 94% rename from spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcItemWriterAutoConfigurationTests.java rename to spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcBatchItemWriterAutoConfigurationTests.java index d1b74a7b..d2c4255f 100644 --- a/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcItemWriterAutoConfigurationTests.java +++ b/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcBatchItemWriterAutoConfigurationTests.java @@ -60,7 +60,7 @@ import org.springframework.util.SocketUtils; import static org.assertj.core.api.Assertions.assertThat; -public class JdbcItemWriterAutoConfigurationTests { +public class JdbcBatchItemWriterAutoConfigurationTests { private final static String DATASOURCE_URL; @@ -99,13 +99,13 @@ public class JdbcItemWriterAutoConfigurationTests { public void baseTest() { ApplicationContextRunner applicationContextRunner = new ApplicationContextRunner() .withUserConfiguration( - JdbcItemWriterAutoConfigurationTests.DelimitedJobConfiguration.class, + JdbcBatchItemWriterAutoConfigurationTests.DelimitedJobConfiguration.class, TaskLauncherConfiguration.class) .withConfiguration( AutoConfigurations.of(PropertyPlaceholderAutoConfiguration.class, BatchAutoConfiguration.class, SingleStepJobAutoConfiguration.class, - JdbcItemWriterAutoConfiguration.class)); + JdbcBatchItemWriterAutoConfiguration.class)); applicationContextRunner = updatePropertiesForTest(applicationContextRunner); runTest(applicationContextRunner); @@ -115,14 +115,14 @@ public class JdbcItemWriterAutoConfigurationTests { public void customSqlParameterSourceTest() { ApplicationContextRunner applicationContextRunner = new ApplicationContextRunner() .withUserConfiguration( - JdbcItemWriterAutoConfigurationTests.DelimitedDifferentKeyNameJobConfiguration.class, + JdbcBatchItemWriterAutoConfigurationTests.DelimitedDifferentKeyNameJobConfiguration.class, TaskLauncherConfiguration.class, CustomSqlParameterSourceProviderConfiguration.class) .withConfiguration( AutoConfigurations.of(PropertyPlaceholderAutoConfiguration.class, BatchAutoConfiguration.class, SingleStepJobAutoConfiguration.class, - JdbcItemWriterAutoConfiguration.class)); + JdbcBatchItemWriterAutoConfiguration.class)); applicationContextRunner = updatePropertiesForTest(applicationContextRunner); runTest(applicationContextRunner); @@ -132,14 +132,14 @@ public class JdbcItemWriterAutoConfigurationTests { public void preparedStatementSetterTest() { ApplicationContextRunner applicationContextRunner = new ApplicationContextRunner() .withUserConfiguration( - JdbcItemWriterAutoConfigurationTests.DelimitedJobConfiguration.class, + JdbcBatchItemWriterAutoConfigurationTests.DelimitedJobConfiguration.class, TaskLauncherConfiguration.class, ItemPreparedStatementSetterConfiguration.class) .withConfiguration( AutoConfigurations.of(PropertyPlaceholderAutoConfiguration.class, BatchAutoConfiguration.class, SingleStepJobAutoConfiguration.class, - JdbcItemWriterAutoConfiguration.class)); + JdbcBatchItemWriterAutoConfiguration.class)); applicationContextRunner = updatePropertiesForTest(applicationContextRunner); runTest(applicationContextRunner); } @@ -148,8 +148,8 @@ public class JdbcItemWriterAutoConfigurationTests { ApplicationContextRunner applicationContextRunner) { return applicationContextRunner.withPropertyValues("spring.batch.job.jobName=job", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.jdbcwriter.name=fooWriter", - "spring.batch.job.jdbcwriter.sql=INSERT INTO item (item_name) VALUES (:item_name)"); + "spring.batch.job.jdbcbatchitemwriter.name=fooWriter", + "spring.batch.job.jdbcbatchitemwriter.sql=INSERT INTO item (item_name) VALUES (:item_name)"); } private void validateResultAndBean(ApplicationContext context) { diff --git a/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcCursorItemReaderAutoConfigurationTests.java b/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcCursorItemReaderAutoConfigurationTests.java index 8239409f..29049289 100644 --- a/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcCursorItemReaderAutoConfigurationTests.java +++ b/spring-cloud-starter-single-step-batch-job/src/test/java/org/springframework/cloud/task/batch/autoconfigure/jdbc/JdbcCursorItemReaderAutoConfigurationTests.java @@ -100,8 +100,8 @@ public class JdbcCursorItemReaderAutoConfigurationTests { JdbcCursorItemReaderAutoConfiguration.class)) .withPropertyValues("spring.batch.job.jobName=integrationJob", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.jdbccursorreader.name=fooReader", - "spring.batch.job.jdbccursorreader.sql=select item_name from item"); + "spring.batch.job.jdbccursoritemreader.name=fooReader", + "spring.batch.job.jdbccursoritemreader.sql=select item_name from item"); applicationContextRunner.run((context) -> { JobLauncher jobLauncher = context.getBean(JobLauncher.class); @@ -138,8 +138,8 @@ public class JdbcCursorItemReaderAutoConfigurationTests { JdbcCursorItemReaderAutoConfiguration.class)) .withPropertyValues("spring.batch.job.jobName=rowMapperJob", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.jdbccursorreader.name=fooReader", - "spring.batch.job.jdbccursorreader.sql=select * from item"); + "spring.batch.job.jdbccursoritemreader.name=fooReader", + "spring.batch.job.jdbccursoritemreader.sql=select * from item"); applicationContextRunner.run((context) -> { JobLauncher jobLauncher = context.getBean(JobLauncher.class); @@ -176,18 +176,18 @@ public class JdbcCursorItemReaderAutoConfigurationTests { JdbcCursorItemReaderAutoConfiguration.class)) .withPropertyValues("spring.batch.job.jobName=roseyJob", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.jdbccursorreader.saveState=false", - "spring.batch.job.jdbccursorreader.name=fooReader", - "spring.batch.job.jdbccursorreader.maxItemCount=15", - "spring.batch.job.jdbccursorreader.currentItemCount=2", - "spring.batch.job.jdbccursorreader.fetchSize=4", - "spring.batch.job.jdbccursorreader.maxRows=6", - "spring.batch.job.jdbccursorreader.queryTimeout=8", - "spring.batch.job.jdbccursorreader.ignoreWarnings=true", - "spring.batch.job.jdbccursorreader.verifyCursorPosition=true", - "spring.batch.job.jdbccursorreader.driverSupportsAbsolute=true", - "spring.batch.job.jdbccursorreader.useSharedExtendedConnection=true", - "spring.batch.job.jdbccursorreader.sql=select * from foo"); + "spring.batch.job.jdbccursoritemreader.saveState=false", + "spring.batch.job.jdbccursoritemreader.name=fooReader", + "spring.batch.job.jdbccursoritemreader.maxItemCount=15", + "spring.batch.job.jdbccursoritemreader.currentItemCount=2", + "spring.batch.job.jdbccursoritemreader.fetchSize=4", + "spring.batch.job.jdbccursoritemreader.maxRows=6", + "spring.batch.job.jdbccursoritemreader.queryTimeout=8", + "spring.batch.job.jdbccursoritemreader.ignoreWarnings=true", + "spring.batch.job.jdbccursoritemreader.verifyCursorPosition=true", + "spring.batch.job.jdbccursoritemreader.driverSupportsAbsolute=true", + "spring.batch.job.jdbccursoritemreader.useSharedExtendedConnection=true", + "spring.batch.job.jdbccursoritemreader.sql=select * from foo"); applicationContextRunner.run((context) -> { @@ -260,7 +260,7 @@ public class JdbcCursorItemReaderAutoConfigurationTests { JdbcCursorItemReaderAutoConfiguration.class)) .withPropertyValues("spring.batch.job.jobName=job", "spring.batch.job.stepName=step1", "spring.batch.job.chunkSize=5", - "spring.batch.job.jdbccursorreader.name=fooReader"); + "spring.batch.job.jdbccursoritemreader.name=fooReader"); assertThatThrownBy(() -> { runTest(applicationContextRunner); diff --git a/spring-cloud-task-docs/src/main/asciidoc/batch-starter.adoc b/spring-cloud-task-docs/src/main/asciidoc/batch-starter.adoc new file mode 100644 index 00000000..400332b0 --- /dev/null +++ b/spring-cloud-task-docs/src/main/asciidoc/batch-starter.adoc @@ -0,0 +1,532 @@ + +[[batch-job-starter]] += Single Step Batch Job Starter + +[[partintro]] +-- +This section goes into how to develop a Spring Batch `Job` with a single `Step` using the +starter included in Spring Cloud Task. This starter allows users to use configuration +to define an `ItemReader`, an `ItemWriter`, or a full single step Spring Batch `Job`. +To read more about Spring Batch and its capabilities, read its documentation +https://spring.io/projects/spring-batch[here]. +-- + +To obtain the starter, add the following to your build: + +Maven: +[source,xml] +---- + + org.springframework.cloud + spring-cloud-starter-single-step-batch-job + 2.3.0 + +---- + +Gradle: +[source,groovy] +---- +compile "org.springframework.cloud:spring-cloud-starter-single-step-batch-job:2.3.0" +---- + +[[job-definition]] +== Defining a Job + +You can use the starter to define as little as an `ItemReader`, `ItemWriter`, or as much as a full `Job`. +In this section, we will define what properties are required to be defined to configure a +`Job`. + +[[job-definition-properties]] +=== Properties + +To begin, the starter provides a set of properties to be able to configure the basics of a Job with one Step. + +.Job Properties +|=== +| Property | Type | Default Value | Description + +| `spring.batch.job.jobName` +| `String` +| `null` +| The name of the job. + +| `spring.batch.job.stepName` +| `String` +| `null` +| The name of the step. + +| `spring.batch.job.chunkSize` +| `Integer` +| `null` +| The number of items to be processed per transaction. +|=== + +With the above properties configured, you will have a job with a single, chunk based step. +This chunk based step will read, process, and write `Map` instances as the +items. However, the step won't do anything yet. You need to configure an `ItemReader`, an +optional `ItemProcessor`, and an `ItemWiter` to give it something to do. To configure one +of these, you can either use properties and configure one of the options that has provided +auto configuration, or you can configure your own via standard Spring configuration +mechanisms. + +NOTE: If you configure your own, the input/output types must match the others in the step. +The `ItemReader` implementations and `ItemWriter` implementations in this starter all use +a `Map` as the input/output item. + +[[item-readers]] +== Autoconfiguration for ItemReader implementations + +This starter provides autoconfiguration for four different `ItemReader` implementations: +`AmqpItemReader`, `FlatFileItemReader`, `JdbcCursorItemReader`, and the `KafkaItemReader`. +In this section we will outline how to configure each of these using the provided +autoconfiguration. + +[[amqpitemreader]] +=== AmqpItemReader + +Reading from a queue or topic via AMQP can be done via the `AmqpItemReader`. The +autoconfiguration for this `ItemReader` implementation is dependent upon two sets of +configuration. The first is the configuration of an `AmqpTemplate`. This can be done either +yourself or via the autoconfiguratino provided by Spring Boot. Documentation for that can +be found https://docs.spring.io/spring-boot/docs/2.4.x/reference/htmlsingle/#boot-features-amqp[here]. +Once the `AmqpTemplate` is configured, enabling the batch capabilities to support it can +be accomplished via the properties below. + +.`AmqpItemReader` Properties +|=== +| Property | Type | Default Value | Description + +| `spring.batch.job.amqpitemreader.enabled` +| `boolean` +| `false` +| If `true`, the autoconfiguration will execute. + +| `spring.batch.job.amqpitemreader.jsonConverterEnabled` +| `boolean` +| `true` +| Indicates if the `Jackson2JsonMessageConverter` should be registered to parse messages. +|=== + +To read more about the `AmqpItemReader`, you can read its documentation https://docs.spring.io/spring-batch/docs/4.3.x/api/org/springframework/batch/item/amqp/AmqpItemReader.html[here] + +[[flatfileitemreader]] +=== FlatFileItemReader + +The `FlatFileItemReader` provides the capability to read from flat files such as CSVs +and other file formats. In order to read from a file, you can provide some components +yourself via normal Spring configuration (`LineTokenizer`, `RecordSeparatorPolicy`, +`FieldSetMapper`, `LineMapper`, or `SkippedLinesCallback`). You can also use the +following properties to configure the reader as required. + +.`FlatFileItemReader` Properties +|=== +| Property | Type | Default Value | Description + +| `spring.batch.job.flatfileitemreader.saveState` +| `boolean` +| `true` +| Determines if the state should be saved for restarts. + +| `spring.batch.job.flatfileitemreader.name` +| `String` +| `null` +| Name used to provide unique keys in the `ExecutionContext`. + +| `spring.batch.job.flatfileitemreader.maxItemcount` +| `int` +| `Integer.MAX_VALUE` +| Maximum number of items to be read from the file. + +| `spring.batch.job.flatfileitemreader.currentItemCount` +| `int` +| 0 +| Number of items that have already been read. Used on restarts. + +| `spring.batch.job.flatfileitemreader.comments` +| `List` +| empty List +| A list of Strings that indicate commented lines (lines to be ignored) in the file. + +| `spring.batch.job.flatfileitemreader.resource` +| `Resource` +| `null` +| The resource to be read. + +| `spring.batch.job.flatfileitemreader.strict` +| `boolean` +| `true` +| If set to true, the reader will throw an exception if the resource is not found. + +| `spring.batch.job.flatfileitemreader.encoding` +| `String` +| `FlatFileItemReader.DEFAULT_CHARSET` +| Encoding to be used when reading the file. + +| `spring.batch.job.flatfileitemreader.linesToSkip` +| `int` +| 0 +| Indicates the number of lines to skip at the start of a file. + +| `spring.batch.job.flatfileitemreader.delimited` +| `boolean` +| `false` +| Indicates if the file is a delimited file (CSV, etc). Only this property or `spring.batch.job.flatfileitemreader.fixedLength` can be `true` at the same time. + +| `spring.batch.job.flatfileitemreader.delimiter` +| `String` +| `DelimitedLineTokenizer.DELIMITER_COMMA` +| If reading a delimited file, indicates the delimiter to parse on. + +| `spring.batch.job.flatfileitemreader.quoteCharacter` +| `char` +| `DelimitedLineTokenizer.DEFAULT_QUOTE_CHARACTER` +| Used to determine what character is used to quote values. + +| `spring.batch.job.flatfileitemreader.includedFields` +| `List` +| empty list +| A list of indicies of which fields in a record to include in the item. + +| `spring.batch.job.flatfileitemreader.fixedLength` +| `boolean` +| `false` +| Indicates if a file's records are parsed via column numbers. Only this property or `spring.batch.job.flatfileitemreader.delimited` can be `true` at the same time. + +| `spring.batch.job.flatfileitemreader.ranges` +| `List` +| empty list +| List of column ranges to parse a fixed width record by. Read about Range https://docs.spring.io/spring-batch/docs/4.3.x/api/org/springframework/batch/item/file/transform/Range.html[here]. + +| `spring.batch.job.flatfileitemreader.names` +| `String []` +| `null` +| List of names for each field parsed from a record. These names are the keys in the `Map` in the items returned from this `ItemReader`. + +| `spring.batch.job.flatfileitemreader.parsingStrict` +| `boolean` +| `true` +| If set to `true`, mapping will fail if fields cannot be mapped. +|=== + +To read more about the `FlatFileItemReader`, you can find its documentation https://docs.spring.io/spring-batch/docs/4.3.x/api/org/springframework/batch/item/file/FlatFileItemReader.html[here]. + +[[jdbcCursorItemReader]] +=== JdbcCursorItemReader + +The `JdbcCursorItemReader` executes a query against a relational database and iterates over +the resulting cursor (`ResultSet`) to provide the resulting items. This autoconfiguration +allows a user to provide a `PreparedStatementSetter` and/or a `RowMapper` if required. They +can also use the properties available to configure a `JdbcCursorItemReader` are as follows. + +.`JdbcCursorItemReader` Properties +|=== +| Property | Type | Default Value | Description + +| `spring.batch.job.jdbccursoritemreader.saveState` +| `boolean` +| `true` +| Determines if the state should be saved for restarts. + +| `spring.batch.job.jdbccursoritemreader.name` +| `String` +| `null` +| Name used to provide unique keys in the `ExecutionContext`. + +| `spring.batch.job.jdbccursoritemreader.maxItemcount` +| `int` +| `Integer.MAX_VALUE` +| Maximum number of items to be read from the file. + +| `spring.batch.job.jdbccursoritemreader.currentItemCount` +| `int` +| 0 +| Number of items that have already been read. Used on restarts. + +| `spring.batch.job.jdbccursoritemreader.fetchSize` +| `int` +| +| A hint to the driver on how many records to retrieve per call to the database system. For best performance, this usually will want to be configured to match the chunk size. + +| `spring.batch.job.jdbccursoritemreader.maxRows` +| `int` +| +| Maximum number of items to read from the database. + +| `spring.batch.job.jdbccursoritemreader.queryTimeout` +| `int` +| +| Number of milliseconds for the query to timeout. + +| `spring.batch.job.jdbccursoritemreader.ignoreWarnings` +| `boolean` +| `true` +| Determines if the reader should ignore SQL warnings when processing. + +| `spring.batch.job.jdbccursoritemreader.verifyCursorPosition` +| `boolean` +| `true` +| Indicates if the cursor's position should be verified after each read to verify that the `RowMapper` did not advance the cursor. + +| `spring.batch.job.jdbccursoritemreader.driverSupportsAbsolute` +| `boolean` +| `false` +| Indicates if the driver supports absolute positioning of a cursor. + +| `spring.batch.job.jdbccursoritemreader.useSharedExtendedConnection` +| `boolean` +| `false` +| Indicates if the connection is shared with other processing (and therefor part of a transaction). + +| `spring.batch.job.jdbccursoritemreader.sql` +| `String` +| `null` +| SQL query to read from. +|=== + +For more information about the `JdbcCursorItemReader`, refer to its documentation https://docs.spring.io/spring-batch/docs/4.3.x/api/org/springframework/batch/item/database/JdbcCursorItemReader.html[here] + +[[kafkaItemReader]] +=== KafkaItemReader + +Ingesting a partition of data from a Kafka topic is useful and exactly what the +`KafkaItemReader` can do. In order to configure a `KafkaItemReader`, two pieces +of configuration are required. First, configuring Kafka via Spring Boot's Kafka +autoconfiguration is required (you can read more about that +https://docs.spring.io/spring-boot/docs/2.4.x/reference/htmlsingle/#boot-features-kafka[here]). +Once the Kafka properties from Spring Boot are configured, the `KafkaItemReader` +itself can be configured via the following properties. + +.`KafkaItemReader` Properties +|=== +| Property | Type | Default Value | Description + +| `spring.batch.job.kafkaitemreader.name` +| `String` +| `null` +| Name used to provide unique keys in the `ExecutionContext`. + +| `spring.batch.job.kafkaitemreader.topic` +| `String` +| `null` +| Name of the topic to read from. + +| `spring.batch.job.kafkaitemreader.partitions` +| `List` +| empty list +| List of partition indicies to read from. + +| `spring.batch.job.kafkaitemreader.pollTimeOutInSeconds` +| `long` +| 30 +| Timeout for the `poll()` operations. + +| `spring.batch.job.kafkaitemreader.saveState` +| `boolean` +| `true` +| Determines if the state should be saved for restarts. +|=== + +You can read more about the `KafkaItemReader` via its documentation +https://docs.spring.io/spring-batch/docs/4.3.x/api/org/springframework/batch/item/kafka/KafkaItemReader.html[here]. + +[[item-processors]] +== ItemProcessor Configuration + +The single step batch job autoconfiguration will accept an `ItemProcessor` if one +is available within the `ApplicationContext`. If one is found of the correct type +(`ItemProcessor, Map>`, it will be autowired +into the step. + +[[item-writers]] +== Autoconfiguration for ItemWriter implementations + +This starter provides autoconfiguration for `ItemWriter` implementations that +match those `ItemReader` implementations supported: `AmqpItemWriter`, +`FlatFileItemWriter`, `JdbcItemWriter`, and `KafkaItemWriter`. This section will +cover how to use the autoconfiguration to configure a supported `ItemWriter`. + +[[amqpitemwriter]] +=== AmqpItemWriter + +To write to a RabbitMQ queue, two sets of configuration are required. First, an +`AmqpTemplate` is required. The easiest way to get this is via Spring Boot's +RabbitMQ autoconfiguration. You can read about Spring Boot's RabbitMQ support https://docs.spring.io/spring-boot/docs/2.4.x/reference/htmlsingle/#boot-features-amqp[here]. +Once the `AmqpTemplate` is configured, you can configure the `AmqpItemWriter` via the +properties below. + +.`AmqpItemWriter` Properties +|=== +| Property | Type | Default Value | Description + +| `spring.batch.job.amqpitemwriter.enabled` +| `boolean` +| `false` +| If `true`, the autoconfiguration will execute. + +| `spring.batch.job.amqpitemwriter.jsonConverterEnabled` +| `boolean` +| `true` +| Indicates if the `Jackson2JsonMessageConverter` should be registered to convert messages. +|=== + +[[flatfileitemwriter]] +=== FlatFileItemWriter + +To write a file as the output of the step, the `FlatFileItemWriter` can be configured. +Autoconfiguration will accept components configured explicitly (like a `LineAggregator`, +`FieldExtractor`, `FlatFileHeaderCallback`, or a `FlatFileFooterCallback`) as well as +be configured using the properties specified below. + +.`FlatFileItemWriter` Properties +|=== +| Property | Type | Default Value | Description + +| `spring.batch.job.flatfileitemwriter.resource` +| `Resource` +| `null` +| The resource to be read. + +| `spring.batch.job.flatfileitemwriter.delimited` +| `boolean` +| `false` +| Indicates if the output file will be a delimited file or not. If `true`, `spring.batch.job.flatfileitemwriter.formatted` must be `false`. + +| `spring.batch.job.flatfileitemwriter.formatted` +| `boolean` +| `false` +| Indicates if the output file will be a formatted file or not. If `true`, `spring.batch.job.flatfileitemwriter.delimited` must be `false`. + +| `spring.batch.job.flatfileitemwriter.format` +| `String` +| `null` +| The format used to generate the output for a formatted file. Formatting performed via `String.format`. + +| `spring.batch.job.flatfileitemwriter.locale` +| `Locale` +| `Locale.getDefault()` +| The Locale to be used when generating the file. + +| `spring.batch.job.flatfileitemwriter.maximumLength` +| `int` +| 0 +| Max length the record can be. If 0, the size is unbound. + +| `spring.batch.job.flatfileitemwriter.minimumLength` +| `int` +| 0 +| The minimum record length. + +| `spring.batch.job.flatfileitemwriter.delimiter` +| `String` +| `,` +| The String used to delimit fields in a delimited file. + +| `spring.batch.job.flatfileitemwriter.encoding` +| `String` +| `FlatFileItemReader.DEFAULT_CHARSET` +| Encoding to be used when writing the file. + +| `spring.batch.job.flatfileitemwriter.forceSync` +| `boolean` +| `false` +| Indicates if a file should be force-synced to the disk on flush. + +| `spring.batch.job.flatfileitemwriter.names` +| `String []` +| `null` +| List of names for each field parsed from a record. These names are the keys in the `Map` in the items received by this `ItemWriter`. + +| `spring.batch.job.flatfileitemwriter.append` +| `boolean` +| `false` +| Indicates if a file should be appended to if the output file is found. + +| `spring.batch.job.flatfileitemwriter.lineSeparator` +| `String` +| `FlatFileItemWriter.DEFAULT_LINE_SEPARATOR` +| What String to use to separate lines in the output file. + +| `spring.batch.job.flatfileitemwriter.name` +| `String` +| `null` +| Name used to provide unique keys in the `ExecutionContext`. + +| `spring.batch.job.flatfileitemwriter.saveState` +| `boolean` +| `true` +| Determines if the state should be saved for restarts. + +| `spring.batch.job.flatfileitemwriter.shouldDeleteIfEmpty` +| `boolean` +| `false` +| If set to true, if there is no output (resulting file is empty) it will be deleted when the job completes. + +| `spring.batch.job.flatfileitemwriter.shouldDeleteIfExists` +| `boolean` +| `true` +| If set to true and a file is found where the output file should be, it will be deleted before the step begins. + +| `spring.batch.job.flatfileitemwriter.transactional` +| `boolean` +| `FlatFileItemWriter.DEFAULT_TRANSACTIONAL` +| Indicates if the reader is a transactional queue (indicating that the items read will be returned to the queue upon a failure). +|=== + +To read more about how to configure the `FlatFileItemWriter` you can refer to its documentation https://docs.spring.io/spring-batch/docs/4.3.x/api/org/springframework/batch/item/file/FlatFileItemWriter.html[here]. + +[[jdbcitemwriter]] +=== JdbcBatchItemWriter + +To write the output of a step to a relational database, this starter provides the ability +to autoconfigure a `JdbcBatchItemWriter`. The autoconfiguration allows a user to provide their +own `ItemPreparedStatementSetter` or `ItemSqlParameterSourceProvider` as well as +configuration options via properties specified below. + +.`JdbcBatchItemWriter` Properties +|=== +| Property | Type | Default Value | Description + +| `spring.batch.job.jdbcbatchitemwriter.name` +| `String` +| `null` +| Name used to provide unique keys in the `ExecutionContext`. + +| `spring.batch.job.jdbcbatchitemwriter.sql` +| `String` +| `null` +| The SQL used to insert each item. + +| `spring.batch.job.jdbcbatchitemwriter.assertUpdates` +| `boolean` +| `true` +| Verify that every insert results in the update of at least one record. +|=== + +To read more about the configuration of the `JdbcBatchItemWriter` you can refer to its documentation https://docs.spring.io/spring-batch/docs/4.3.x/api/org/springframework/batch/item/database/JdbcBatchItemWriter.html[here] + +[[kafkaitemwriter]] +=== KafkaItemWriter + +To write step output to a Kafka topic, a `KafkaItemWriter` is required. This starter +provides autoconfiguration for a `KafkaItemWriter` using facilities from two places. +First, Spring Boot's Kafka autoconfiguration. You can read more about it +https://docs.spring.io/spring-boot/docs/2.4.x/reference/htmlsingle/#boot-features-kafka[here]. +Second, there are two properties that this starter allows you to configure on the writer. + +.`KafkaItemWriter` Properties +|=== +| Property | Type | Default Value | Description + +| `spring.batch.job.kafkaitemwriter.topic` +| `String` +| `null` +| The Kafka topic to write to. + +| `spring.batch.job.kafkaitemwriter.delete` +| `boolean` +| `false` +| Indicates if the items being passed to the writer are all to be sent as delete events to the topic. +|=== + +To read more about the configuration options for the `KafkaItemWiter`, you can read +its documentation https://docs.spring.io/spring-batch/docs/4.3.x/api/org/springframework/batch/item/kafka/KafkaItemWriter.html[here]. diff --git a/spring-cloud-task-docs/src/main/asciidoc/index.adoc b/spring-cloud-task-docs/src/main/asciidoc/index.adoc index c565a018..0648d746 100644 --- a/spring-cloud-task-docs/src/main/asciidoc/index.adoc +++ b/spring-cloud-task-docs/src/main/asciidoc/index.adoc @@ -54,6 +54,8 @@ include::features.adoc[] include::batch.adoc[] +include::batch-starter.adoc[] + include::stream.adoc[] include::appendix.adoc[] diff --git a/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-ffreader.properties b/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-ffreader.properties index 1c776305..a129f015 100644 --- a/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-ffreader.properties +++ b/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-ffreader.properties @@ -1,9 +1,9 @@ -spring.batch.job.flatfilereader.savestate=true -spring.batch.job.flatfilereader.name=fixedWidthConfiguration -spring.batch.job.flatfilereader.comments=#,$ -spring.batch.job.flatfilereader.resource=/test.txt -spring.batch.job.flatfilereader.strict=true -spring.batch.job.flatfilereader.fixedLength=true -spring.batch.job.flatfilereader.ranges=1-3 -spring.batch.job.flatfilereader.names=ITEM_NAME -spring.batch.job.flatfilereader.parsingStrict=false +spring.batch.job.flatfileitemreader.savestate=true +spring.batch.job.flatfileitemreader.name=fixedWidthConfiguration +spring.batch.job.flatfileitemreader.comments=#,$ +spring.batch.job.flatfileitemreader.resource=/test.txt +spring.batch.job.flatfileitemreader.strict=true +spring.batch.job.flatfileitemreader.fixedLength=true +spring.batch.job.flatfileitemreader.ranges=1-3 +spring.batch.job.flatfileitemreader.names=ITEM_NAME +spring.batch.job.flatfileitemreader.parsingStrict=false diff --git a/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-ffwriter.properties b/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-ffwriter.properties index bd2f7ef9..06b391f3 100644 --- a/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-ffwriter.properties +++ b/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-ffwriter.properties @@ -1,11 +1,11 @@ -spring.batch.job.flatfilewriter.name=fooWriter -spring.batch.job.flatfilewriter.resource=file:result.txt -spring.batch.job.flatfilewriter.encoding=UTF-16 -spring.batch.job.flatfilewriter.saveState=false -spring.batch.job.flatfilewriter.shouldDeleteIfEmpty=true -spring.batch.job.flatfilewriter.delimited=true -spring.batch.job.flatfilewriter.names=ITEM_NAME -spring.batch.job.flatfilewriter.append=true -spring.batch.job.flatfilewriter.forceSync=true -spring.batch.job.flatfilewriter.shouldDeleteIfExists=false -spring.batch.job.flatfilewriter.transactional=false +spring.batch.job.flatfileitemwriter.name=fooWriter +spring.batch.job.flatfileitemwriter.resource=file:result.txt +spring.batch.job.flatfileitemwriter.encoding=UTF-16 +spring.batch.job.flatfileitemwriter.saveState=false +spring.batch.job.flatfileitemwriter.shouldDeleteIfEmpty=true +spring.batch.job.flatfileitemwriter.delimited=true +spring.batch.job.flatfileitemwriter.names=ITEM_NAME +spring.batch.job.flatfileitemwriter.append=true +spring.batch.job.flatfileitemwriter.forceSync=true +spring.batch.job.flatfileitemwriter.shouldDeleteIfExists=false +spring.batch.job.flatfileitemwriter.transactional=false diff --git a/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-jdbcreader.properties b/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-jdbcreader.properties index c94a43c0..67e55089 100644 --- a/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-jdbcreader.properties +++ b/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-jdbcreader.properties @@ -1,2 +1,2 @@ -spring.batch.job.jdbccursorreader.name=fooReader -spring.batch.job.jdbccursorreader.sql=select item_name from item_sample order by item_name +spring.batch.job.jdbccursoritemreader.name=fooReader +spring.batch.job.jdbccursoritemreader.sql=select item_name from item_sample order by item_name diff --git a/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-jdbcwriter.properties b/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-jdbcwriter.properties index 93aa0d38..aa5fc349 100644 --- a/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-jdbcwriter.properties +++ b/spring-cloud-task-samples/single-step-batch-job/src/main/resources/application-jdbcwriter.properties @@ -1,2 +1,2 @@ -spring.batch.job.jdbcwriter.name=jdbcWriter -spring.batch.job.jdbcwriter.sql=INSERT INTO item (item_name) VALUES (:ITEM_NAME) +spring.batch.job.jdbcbatchitemwriter.name=jdbcWriter +spring.batch.job.jdbcbatchitemwriter.sql=INSERT INTO item (item_name) VALUES (:ITEM_NAME)