From b1ac90df68e54b6a6cbd6ae1d832e40b37d8fc0e Mon Sep 17 00:00:00 2001 From: Mahmoud Ben Hassine Date: Wed, 4 Oct 2023 17:46:36 +0200 Subject: [PATCH] Add java configuration for the multi-record type sample Issue #3663 --- spring-batch-samples/README.md | 23 +++ .../DelegatingTradeLineAggregator.java | 4 +- .../MultiRecordTypeJobConfiguration.java | 156 ++++++++++++++++++ .../sample/file/multirecordtype/README.md | 35 ++++ .../multirecordtype/data}/multiRecordType.txt | 0 .../multirecordtype/job}/multiRecordType.xml | 10 +- .../MultiRecordTypeFunctionalTests.java | 96 +++++++++++ .../MultiRecordTypeFunctionalTests.java | 61 ------- 8 files changed, 317 insertions(+), 68 deletions(-) rename spring-batch-samples/src/{test/java/org/springframework/batch/sample/iosample/internal => main/java/org/springframework/batch/sample/file/multirecordtype}/DelegatingTradeLineAggregator.java (93%) create mode 100644 spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multirecordtype/MultiRecordTypeJobConfiguration.java create mode 100644 spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multirecordtype/README.md rename spring-batch-samples/src/main/resources/{data/iosample/input => org/springframework/batch/sample/file/multirecordtype/data}/multiRecordType.txt (100%) rename spring-batch-samples/src/main/resources/{jobs/iosample => org/springframework/batch/sample/file/multirecordtype/job}/multiRecordType.xml (90%) create mode 100644 spring-batch-samples/src/test/java/org/springframework/batch/sample/file/multirecordtype/MultiRecordTypeFunctionalTests.java delete mode 100644 spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/MultiRecordTypeFunctionalTests.java diff --git a/spring-batch-samples/README.md b/spring-batch-samples/README.md index 3a89fbccf..23e6d8c77 100644 --- a/spring-batch-samples/README.md +++ b/spring-batch-samples/README.md @@ -230,6 +230,29 @@ END [MultiLine Input Job Sample](./src/main/java/org/springframework/batch/sample/file/multiline/README.md) +### MultiRecord type Input Job + +The goal of this sample is to show how to use the `PatternMatchingCompositeLineMapper` API +to process files containing lines of different types: + +``` +CUST42001customer100012000 +CUST42002customer200022000 +CUST42003customer300032000 +TRADUK21341EAH45978 98.34customer1 +TRADUK21341EAH46112 18.12customer2 +CUST42004customer400042000 +CUST42005customer500052000 +TRADUK21341EAH47245 12.78customer3 +TRADUK21341EAH48108109.25customer4 +TRADUK21341EAH49854123.39customer5 +CUST42006customer600062000 +TRADUK21341EAH50234 32.45customer6 +... +``` + +[MultiRecord type Input Job Sample](./src/main/java/org/springframework/batch/sample/file/multirecordtype/README.md) + ### Football Job This is a (American) Football statistics loading job. It loads two files containing players and games diff --git a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/internal/DelegatingTradeLineAggregator.java b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multirecordtype/DelegatingTradeLineAggregator.java similarity index 93% rename from spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/internal/DelegatingTradeLineAggregator.java rename to spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multirecordtype/DelegatingTradeLineAggregator.java index b1707b05a..c93a93cef 100644 --- a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/internal/DelegatingTradeLineAggregator.java +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multirecordtype/DelegatingTradeLineAggregator.java @@ -1,5 +1,5 @@ /* - * Copyright 2006-2014 the original author or authors. + * Copyright 2006-2023 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.batch.sample.iosample.internal; +package org.springframework.batch.sample.file.multirecordtype; import org.springframework.batch.item.file.transform.LineAggregator; import org.springframework.batch.sample.domain.trade.CustomerCredit; diff --git a/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multirecordtype/MultiRecordTypeJobConfiguration.java b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multirecordtype/MultiRecordTypeJobConfiguration.java new file mode 100644 index 000000000..613555aa3 --- /dev/null +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multirecordtype/MultiRecordTypeJobConfiguration.java @@ -0,0 +1,156 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.sample.file.multirecordtype; + +import java.util.Map; + +import javax.sql.DataSource; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.item.file.FlatFileItemReader; +import org.springframework.batch.item.file.FlatFileItemWriter; +import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder; +import org.springframework.batch.item.file.builder.FlatFileItemWriterBuilder; +import org.springframework.batch.item.file.mapping.PatternMatchingCompositeLineMapper; +import org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor; +import org.springframework.batch.item.file.transform.FixedLengthTokenizer; +import org.springframework.batch.item.file.transform.FormatterLineAggregator; +import org.springframework.batch.item.file.transform.Range; +import org.springframework.batch.sample.domain.trade.CustomerCredit; +import org.springframework.batch.sample.domain.trade.Trade; +import org.springframework.batch.sample.domain.trade.internal.CustomerCreditFieldSetMapper; +import org.springframework.batch.sample.domain.trade.internal.TradeFieldSetMapper; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.Resource; +import org.springframework.core.io.WritableResource; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.jdbc.support.JdbcTransactionManager; + +/** + * @author Mahmoud Ben Hassine + */ +@Configuration +@EnableBatchProcessing +public class MultiRecordTypeJobConfiguration { + + @Bean + @StepScope + public FlatFileItemReader itemReader(PatternMatchingCompositeLineMapper lineMapper, + @Value("#{jobParameters[inputFile]}") Resource resource) { + return new FlatFileItemReaderBuilder().name("itemReader").resource(resource).lineMapper(lineMapper).build(); + } + + @Bean + public PatternMatchingCompositeLineMapper prefixMatchingLineMapper() { + PatternMatchingCompositeLineMapper mapper = new PatternMatchingCompositeLineMapper(); + mapper.setTokenizers(Map.of("TRAD*", tradeLineTokenizer(), "CUST*", customerLineTokenizer())); + mapper.setFieldSetMappers( + Map.of("TRAD*", new TradeFieldSetMapper(), "CUST*", new CustomerCreditFieldSetMapper())); + return mapper; + } + + @Bean + public FixedLengthTokenizer tradeLineTokenizer() { + FixedLengthTokenizer tokenizer = new FixedLengthTokenizer(); + tokenizer.setNames("isin", "quantity", "price", "customer"); + tokenizer.setColumns(new Range(5, 16), new Range(17, 19), new Range(20, 25), new Range(26, 34)); + return tokenizer; + } + + @Bean + public FixedLengthTokenizer customerLineTokenizer() { + FixedLengthTokenizer tokenizer = new FixedLengthTokenizer(); + tokenizer.setNames("id", "name", "credit"); + tokenizer.setColumns(new Range(5, 9), new Range(10, 18), new Range(19, 26)); + return tokenizer; + } + + @Bean + @StepScope + public FlatFileItemWriter itemWriter(DelegatingTradeLineAggregator delegatingTradeLineAggregator, + @Value("#{jobParameters[outputFile]}") WritableResource resource) { + return new FlatFileItemWriterBuilder().name("iemWriter") + .resource(resource) + .lineAggregator(delegatingTradeLineAggregator) + .build(); + } + + @Bean + public DelegatingTradeLineAggregator delegatingTradeLineAggregator( + FormatterLineAggregator tradeLineAggregator, + FormatterLineAggregator customerLineAggregator) { + DelegatingTradeLineAggregator lineAggregator = new DelegatingTradeLineAggregator(); + lineAggregator.setTradeLineAggregator(tradeLineAggregator); + lineAggregator.setCustomerLineAggregator(customerLineAggregator); + return lineAggregator; + } + + @Bean + public FormatterLineAggregator tradeLineAggregator() { + FormatterLineAggregator formatterLineAggregator = new FormatterLineAggregator<>(); + BeanWrapperFieldExtractor fieldExtractor = new BeanWrapperFieldExtractor<>(); + fieldExtractor.setNames(new String[] { "isin", "quantity", "price", "customer" }); + formatterLineAggregator.setFieldExtractor(fieldExtractor); + formatterLineAggregator.setFormat("TRAD%-12s%-3d%6s%-9s"); + return formatterLineAggregator; + } + + @Bean + public FormatterLineAggregator customerLineAggregator() { + FormatterLineAggregator formatterLineAggregator = new FormatterLineAggregator<>(); + BeanWrapperFieldExtractor fieldExtractor = new BeanWrapperFieldExtractor<>(); + fieldExtractor.setNames(new String[] { "id", "name", "credit" }); + formatterLineAggregator.setFieldExtractor(fieldExtractor); + formatterLineAggregator.setFormat("CUST%05d%-9s%08.0f"); + return formatterLineAggregator; + } + + @Bean + public Job job(JobRepository jobRepository, JdbcTransactionManager transactionManager, + FlatFileItemReader itemReader, FlatFileItemWriter itemWriter) { + return new JobBuilder("ioSampleJob", jobRepository) + .start(new StepBuilder("step1", jobRepository).chunk(2, transactionManager) + .reader(itemReader) + .writer(itemWriter) + .build()) + .build(); + } + + // Infrastructure beans + + @Bean + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder().setType(EmbeddedDatabaseType.HSQL) + .addScript("/org/springframework/batch/core/schema-drop-hsqldb.sql") + .addScript("/org/springframework/batch/core/schema-hsqldb.sql") + .generateUniqueName(true) + .build(); + } + + @Bean + public JdbcTransactionManager transactionManager(DataSource dataSource) { + return new JdbcTransactionManager(dataSource); + } + +} diff --git a/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multirecordtype/README.md b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multirecordtype/README.md new file mode 100644 index 000000000..d124dbf88 --- /dev/null +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multirecordtype/README.md @@ -0,0 +1,35 @@ +### MultiRecord type Input Job + +## About + +The goal of this sample is to show how to use the `PatternMatchingCompositeLineMapper` API +to process files containing lines of different types: + +``` +CUST42001customer100012000 +CUST42002customer200022000 +CUST42003customer300032000 +TRADUK21341EAH45978 98.34customer1 +TRADUK21341EAH46112 18.12customer2 +CUST42004customer400042000 +CUST42005customer500052000 +TRADUK21341EAH47245 12.78customer3 +TRADUK21341EAH48108109.25customer4 +TRADUK21341EAH49854123.39customer5 +CUST42006customer600062000 +TRADUK21341EAH50234 32.45customer6 +... +``` + +## Run the sample + +You can run the sample from the command line as following: + +``` +$>cd spring-batch-samples +# Launch the sample using the XML configuration +$>../mvnw -Dtest=MultiRecordTypeFunctionalTests#testLaunchJobWithXmlConfig test +# Launch the sample using the Java configuration +$>../mvnw -Dtest=MultiRecordTypeFunctionalTests#testLaunchJobWithJavaConfig test +``` + diff --git a/spring-batch-samples/src/main/resources/data/iosample/input/multiRecordType.txt b/spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/multirecordtype/data/multiRecordType.txt similarity index 100% rename from spring-batch-samples/src/main/resources/data/iosample/input/multiRecordType.txt rename to spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/multirecordtype/data/multiRecordType.txt diff --git a/spring-batch-samples/src/main/resources/jobs/iosample/multiRecordType.xml b/spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/multirecordtype/job/multiRecordType.xml similarity index 90% rename from spring-batch-samples/src/main/resources/jobs/iosample/multiRecordType.xml rename to spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/multirecordtype/job/multiRecordType.xml index 101367ba9..c7c4db76c 100644 --- a/spring-batch-samples/src/main/resources/jobs/iosample/multiRecordType.xml +++ b/spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/multirecordtype/job/multiRecordType.xml @@ -14,8 +14,8 @@ - - + + @@ -54,10 +54,10 @@ class="org.springframework.batch.sample.domain.trade.internal.CustomerCreditFieldSetMapper" /> - - + + - + diff --git a/spring-batch-samples/src/test/java/org/springframework/batch/sample/file/multirecordtype/MultiRecordTypeFunctionalTests.java b/spring-batch-samples/src/test/java/org/springframework/batch/sample/file/multirecordtype/MultiRecordTypeFunctionalTests.java new file mode 100644 index 000000000..bd72a6c93 --- /dev/null +++ b/spring-batch-samples/src/test/java/org/springframework/batch/sample/file/multirecordtype/MultiRecordTypeFunctionalTests.java @@ -0,0 +1,96 @@ +/* + * Copyright 2006-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.sample.file.multirecordtype; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import org.springframework.batch.core.BatchStatus; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.sample.file.multiline.MultiLineJobConfiguration; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.FileSystemResource; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author Dan Garrette + * @author Mahmoud Ben Hassine + * @author Glenn Renfro + * @since 2.0 + */ +@SpringJUnitConfig(locations = { "/org/springframework/batch/sample/file/multirecordtype/job/multiRecordType.xml", + "/simple-job-launcher-context.xml", "/job-runner-context.xml" }) +class MultiRecordTypeFunctionalTests { + + private static final String OUTPUT_FILE = "target/test-outputs/multiRecordTypeOutput.txt"; + + private static final String INPUT_FILE = "org/springframework/batch/sample/file/multirecordtype/data/multiRecordType.txt"; + + @Autowired + private JobLauncherTestUtils jobLauncherTestUtils; + + @Test + void testLaunchJobWithXmlConfig() throws Exception { + // given + JobParameters jobParameters = new JobParametersBuilder().addString("inputFile", INPUT_FILE) + .addString("outputFile", "file:./" + OUTPUT_FILE) + .toJobParameters(); + + // when + JobExecution jobExecution = this.jobLauncherTestUtils.launchJob(jobParameters); + + // then + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + Path inputFile = new ClassPathResource(INPUT_FILE).getFile().toPath(); + Path outputFile = new FileSystemResource(OUTPUT_FILE).getFile().toPath(); + Assertions.assertLinesMatch(Files.lines(inputFile), Files.lines(outputFile)); + } + + @Test + public void testLaunchJobWithJavaConfig() throws Exception { + // given + ApplicationContext context = new AnnotationConfigApplicationContext(MultiRecordTypeJobConfiguration.class); + JobLauncher jobLauncher = context.getBean(JobLauncher.class); + Job job = context.getBean(Job.class); + JobParameters jobParameters = new JobParametersBuilder().addString("inputFile", INPUT_FILE) + .addString("outputFile", "file:./" + OUTPUT_FILE) + .toJobParameters(); + + // when + JobExecution jobExecution = jobLauncher.run(job, jobParameters); + + // then + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + Path inputFile = new ClassPathResource(INPUT_FILE).getFile().toPath(); + Path outputFile = new FileSystemResource(OUTPUT_FILE).getFile().toPath(); + Assertions.assertLinesMatch(Files.lines(inputFile), Files.lines(outputFile)); + } + +} diff --git a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/MultiRecordTypeFunctionalTests.java b/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/MultiRecordTypeFunctionalTests.java deleted file mode 100644 index bce410951..000000000 --- a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/MultiRecordTypeFunctionalTests.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2006-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.batch.sample.iosample; - -import java.nio.file.Files; -import java.nio.file.Path; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.FileSystemResource; -import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; - -/** - * @author Dan Garrette - * @author Mahmoud Ben Hassine - * @author Glenn Renfro - * @since 2.0 - */ -@SpringJUnitConfig(locations = { "/simple-job-launcher-context.xml", "/jobs/iosample/multiRecordType.xml", - "/job-runner-context.xml" }) -class MultiRecordTypeFunctionalTests { - - private static final String OUTPUT_FILE = "target/test-outputs/multiRecordTypeOutput.txt"; - - private static final String INPUT_FILE = "src/main/resources/data/iosample/input/multiRecordType.txt"; - - @Autowired - private JobLauncherTestUtils jobLauncherTestUtils; - - /** - * Output should be the same as input - */ - @Test - void testJob() throws Exception { - // when - jobLauncherTestUtils.launchJob(); - - // then - Path inputFile = new FileSystemResource(INPUT_FILE).getFile().toPath(); - Path outputFile = new FileSystemResource(OUTPUT_FILE).getFile().toPath(); - Assertions.assertLinesMatch(Files.lines(inputFile), Files.lines(outputFile)); - } - -}