From 27502384629a849c2457fe16e1b5f41dce288b13 Mon Sep 17 00:00:00 2001 From: Mahmoud Ben Hassine Date: Tue, 26 Sep 2023 09:08:02 +0200 Subject: [PATCH] Add java configuration for the delimited file import sample Issue #3663 --- spring-batch-samples/README.md | 7 ++ .../delimited/DelimitedJobConfiguration.java | 82 ++++++++++++++ .../batch/sample/file/delimited/README.md | 19 ++++ .../data/iosample/input/delimited.csv | 10 +- .../data/iosample/input/delimited2.csv | 2 +- .../sample/file/delimited/data/delimited.csv | 6 + .../sample/file/delimited/job}/delimited.xml | 19 +++- .../delimited/DelimitedFunctionalTests.java | 82 ++++++++++++++ .../iosample/DelimitedFunctionalTests.java | 54 --------- ...oJobInstancesDelimitedFunctionalTests.java | 107 ------------------ 10 files changed, 218 insertions(+), 170 deletions(-) create mode 100644 spring-batch-samples/src/main/java/org/springframework/batch/sample/file/delimited/DelimitedJobConfiguration.java create mode 100644 spring-batch-samples/src/main/java/org/springframework/batch/sample/file/delimited/README.md create mode 100644 spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/delimited/data/delimited.csv rename spring-batch-samples/src/main/resources/{jobs/iosample => org/springframework/batch/sample/file/delimited/job}/delimited.xml (68%) create mode 100644 spring-batch-samples/src/test/java/org/springframework/batch/sample/file/delimited/DelimitedFunctionalTests.java delete mode 100644 spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/DelimitedFunctionalTests.java delete mode 100644 spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/TwoJobInstancesDelimitedFunctionalTests.java diff --git a/spring-batch-samples/README.md b/spring-batch-samples/README.md index 008df335c..1504259e0 100644 --- a/spring-batch-samples/README.md +++ b/spring-batch-samples/README.md @@ -162,6 +162,13 @@ This sample shows the delegate pattern again, and also the `ItemReaderAdapter` which is used to adapt a POJO to the `ItemReader` interface. +### Delimited File Import Job + +The goal is to demonstrate a typical scenario of reading data +from a delimited file, processing it and writing it to another file. + +[Delimited file Job sample](./src/main/java/org/springframework/batch/sample/file/delimited/README.md) + ### Fixed Length Import Job The goal is to demonstrate a typical scenario of importing data diff --git a/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/delimited/DelimitedJobConfiguration.java b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/delimited/DelimitedJobConfiguration.java new file mode 100644 index 000000000..3c9a0333e --- /dev/null +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/delimited/DelimitedJobConfiguration.java @@ -0,0 +1,82 @@ +package org.springframework.batch.sample.file.delimited; + +import javax.sql.DataSource; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.file.FlatFileItemReader; +import org.springframework.batch.item.file.FlatFileItemWriter; +import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder; +import org.springframework.batch.item.file.builder.FlatFileItemWriterBuilder; +import org.springframework.batch.sample.domain.trade.CustomerCredit; +import org.springframework.batch.sample.domain.trade.internal.CustomerCreditIncreaseProcessor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.Resource; +import org.springframework.core.io.WritableResource; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.jdbc.support.JdbcTransactionManager; + +@Configuration +@EnableBatchProcessing +public class DelimitedJobConfiguration { + + @Bean + @StepScope + public FlatFileItemReader itemReader(@Value("#{jobParameters[inputFile]}") Resource resource) { + return new FlatFileItemReaderBuilder().name("itemReader") + .resource(resource) + .delimited() + .names("name", "credit") + .targetType(CustomerCredit.class) + .build(); + } + + @Bean + @StepScope + public FlatFileItemWriter itemWriter( + @Value("#{jobParameters[outputFile]}") WritableResource resource) { + return new FlatFileItemWriterBuilder().name("itemWriter") + .resource(resource) + .delimited() + .names("name", "credit") + .build(); + } + + @Bean + public Job job(JobRepository jobRepository, JdbcTransactionManager transactionManager, + ItemReader itemReader, ItemWriter itemWriter) { + return new JobBuilder("ioSampleJob", jobRepository) + .start(new StepBuilder("step1", jobRepository).chunk(2, transactionManager) + .reader(itemReader) + .processor(new CustomerCreditIncreaseProcessor()) + .writer(itemWriter) + .build()) + .build(); + } + + // Infrastructure beans + + @Bean + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder().setType(EmbeddedDatabaseType.HSQL) + .addScript("/org/springframework/batch/core/schema-drop-hsqldb.sql") + .addScript("/org/springframework/batch/core/schema-hsqldb.sql") + .generateUniqueName(true) + .build(); + } + + @Bean + public JdbcTransactionManager transactionManager(DataSource dataSource) { + return new JdbcTransactionManager(dataSource); + } + +} diff --git a/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/delimited/README.md b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/delimited/README.md new file mode 100644 index 000000000..8fea13427 --- /dev/null +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/delimited/README.md @@ -0,0 +1,19 @@ +### Delimited File Import Job + +## About + +The goal is to demonstrate a typical scenario of reading data +from a delimited file, processing it and writing it to another file. + +## Run the sample + +You can run the sample from the command line as following: + +``` +$>cd spring-batch-samples +# Launch the sample using the XML configuration +$>../mvnw -Dtest=DelimitedFunctionalTests#testLaunchJobWithXmlConfig test +# Launch the sample using the Java configuration +$>../mvnw -Dtest=DelimitedFunctionalTests#testLaunchJobWithJavaConfig test +``` + diff --git a/spring-batch-samples/src/main/resources/data/iosample/input/delimited.csv b/spring-batch-samples/src/main/resources/data/iosample/input/delimited.csv index 8c8c3989e..95a077e6b 100644 --- a/spring-batch-samples/src/main/resources/data/iosample/input/delimited.csv +++ b/spring-batch-samples/src/main/resources/data/iosample/input/delimited.csv @@ -1,6 +1,6 @@ -customer1,10 -customer2,20 -customer3,30 -customer4,40 -customer5,50 +customer1,10 +customer2,20 +customer3,30 +customer4,40 +customer5,50 customer6,60 \ No newline at end of file diff --git a/spring-batch-samples/src/main/resources/data/iosample/input/delimited2.csv b/spring-batch-samples/src/main/resources/data/iosample/input/delimited2.csv index 59bce0497..46f251216 100644 --- a/spring-batch-samples/src/main/resources/data/iosample/input/delimited2.csv +++ b/spring-batch-samples/src/main/resources/data/iosample/input/delimited2.csv @@ -1,2 +1,2 @@ -customer7,70 +customer7,70 customer8,80 \ No newline at end of file diff --git a/spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/delimited/data/delimited.csv b/spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/delimited/data/delimited.csv new file mode 100644 index 000000000..95a077e6b --- /dev/null +++ b/spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/delimited/data/delimited.csv @@ -0,0 +1,6 @@ +customer1,10 +customer2,20 +customer3,30 +customer4,40 +customer5,50 +customer6,60 \ No newline at end of file diff --git a/spring-batch-samples/src/main/resources/jobs/iosample/delimited.xml b/spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/delimited/job/delimited.xml similarity index 68% rename from spring-batch-samples/src/main/resources/jobs/iosample/delimited.xml rename to spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/delimited/job/delimited.xml index 084991ee7..d398e1f73 100644 --- a/spring-batch-samples/src/main/resources/jobs/iosample/delimited.xml +++ b/spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/delimited/job/delimited.xml @@ -1,9 +1,22 @@ + + + + + + + + + + @@ -30,7 +43,7 @@ - + diff --git a/spring-batch-samples/src/test/java/org/springframework/batch/sample/file/delimited/DelimitedFunctionalTests.java b/spring-batch-samples/src/test/java/org/springframework/batch/sample/file/delimited/DelimitedFunctionalTests.java new file mode 100644 index 000000000..099e44247 --- /dev/null +++ b/spring-batch-samples/src/test/java/org/springframework/batch/sample/file/delimited/DelimitedFunctionalTests.java @@ -0,0 +1,82 @@ +/* + * Copyright 2006-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.sample.file.delimited; + +import org.junit.jupiter.api.Test; + +import org.springframework.batch.core.BatchStatus; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author Dan Garrette + * @author Dave Syer + * @author Glenn Renfro + * @author Mahmoud Ben Hassine + * @since 2.0 + */ +@SpringJUnitConfig(locations = { "/org/springframework/batch/sample/file/delimited/job/delimited.xml", + "/simple-job-launcher-context.xml", "/job-runner-context.xml" }) +class DelimitedFunctionalTests { + + @Autowired + private JobLauncherTestUtils jobLauncherTestUtils; + + @Test + void testLaunchJobWithXmlConfig() throws Exception { + // given + JobParameters jobParameters = new JobParametersBuilder() + .addString("inputFile", "org/springframework/batch/sample/file/delimited/data/delimited.csv") + .addString("outputFile", "file:./target/test-outputs/delimitedOutput.csv") + .toJobParameters(); + + // when + JobExecution jobExecution = this.jobLauncherTestUtils.launchJob(jobParameters); + + // then + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + } + + @Test + public void testLaunchJobWithJavaConfig() throws Exception { + // given + ApplicationContext context = new AnnotationConfigApplicationContext(DelimitedJobConfiguration.class); + JobLauncher jobLauncher = context.getBean(JobLauncher.class); + Job job = context.getBean(Job.class); + JobParameters jobParameters = new JobParametersBuilder() + .addString("inputFile", "org/springframework/batch/sample/file/delimited/data/delimited.csv") + .addString("outputFile", "file:./target/test-outputs/delimitedOutput.csv") + .toJobParameters(); + + // when + JobExecution jobExecution = jobLauncher.run(job, jobParameters); + + // then + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + } + +} \ No newline at end of file diff --git a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/DelimitedFunctionalTests.java b/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/DelimitedFunctionalTests.java deleted file mode 100644 index 92b47ff4a..000000000 --- a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/DelimitedFunctionalTests.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2006-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.batch.sample.iosample; - -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.StepExecution; -import org.springframework.batch.core.scope.context.StepSynchronizationManager; -import org.springframework.batch.item.ItemReader; -import org.springframework.batch.sample.domain.trade.CustomerCredit; -import org.springframework.batch.test.MetaDataInstanceFactory; -import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; - -/** - * @author Dan Garrette - * @author Dave Syer - * @author Glenn Renfro - * @author Mahmoud Ben Hassine - * @since 2.0 - */ -@SpringJUnitConfig(locations = "/jobs/iosample/delimited.xml") -class DelimitedFunctionalTests extends AbstractIoSampleTests { - - @Override - protected void pointReaderToOutput(ItemReader reader) { - JobParameters jobParameters = super.getUniqueJobParametersBuilder() - .addString("inputFile", "file:./target/test-outputs/delimitedOutput.csv") - .toJobParameters(); - StepExecution stepExecution = MetaDataInstanceFactory.createStepExecution(jobParameters); - StepSynchronizationManager.close(); - StepSynchronizationManager.register(stepExecution); - } - - @Override - protected JobParameters getUniqueJobParameters() { - return super.getUniqueJobParametersBuilder().addString("inputFile", "data/iosample/input/delimited.csv") - .addString("outputFile", "file:./target/test-outputs/delimitedOutput.csv") - .toJobParameters(); - } - -} \ No newline at end of file diff --git a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/TwoJobInstancesDelimitedFunctionalTests.java b/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/TwoJobInstancesDelimitedFunctionalTests.java deleted file mode 100644 index dd485ad21..000000000 --- a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/TwoJobInstancesDelimitedFunctionalTests.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2006-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.batch.sample.iosample; - -import java.util.Date; - -import org.junit.jupiter.api.Test; - -import org.springframework.batch.core.BatchStatus; -import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.JobParametersBuilder; -import org.springframework.batch.core.StepExecution; -import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.item.ExecutionContext; -import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.ItemStream; -import org.springframework.batch.sample.domain.trade.CustomerCredit; -import org.springframework.batch.test.MetaDataInstanceFactory; -import org.springframework.batch.test.StepScopeTestUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author Dave Syer - * @author Glenn Renfro - * @author Mahmoud Ben Hassine - * @since 2.0 - */ -@SpringJUnitConfig( - locations = { "/simple-job-launcher-context.xml", "/jobs/ioSampleJob.xml", "/jobs/iosample/delimited.xml" }) -class TwoJobInstancesDelimitedFunctionalTests { - - @Autowired - private JobLauncher launcher; - - @Autowired - private Job job; - - @Autowired - private ItemReader reader; - - @Autowired - @Qualifier("itemReader") - private ItemStream readerStream; - - @Test - void testLaunchJobTwice() throws Exception { - JobExecution jobExecution = launcher.run(this.job, getJobParameters("data/iosample/input/delimited.csv")); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - verifyOutput(6); - jobExecution = launcher.run(this.job, getJobParameters("data/iosample/input/delimited2.csv")); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - verifyOutput(2); - } - - private void verifyOutput(int expected) throws Exception { - JobParameters jobParameters = new JobParametersBuilder() - .addString("inputFile", "file:./target/test-outputs/delimitedOutput.csv") - .toJobParameters(); - StepExecution stepExecution = MetaDataInstanceFactory.createStepExecution(jobParameters); - - int count = StepScopeTestUtils.doInStepScope(stepExecution, () -> { - int count1 = 0; - - readerStream.open(new ExecutionContext()); - - try { - while (reader.read() != null) { - count1++; - } - } - finally { - readerStream.close(); - } - return count1; - }); - - assertEquals(expected, count); - } - - protected JobParameters getJobParameters(String fileName) { - return new JobParametersBuilder().addLong("timestamp", new Date().getTime()) - .addString("inputFile", fileName) - .addString("outputFile", "file:./target/test-outputs/delimitedOutput.csv") - .toJobParameters(); - } - -}