diff --git a/spring-batch-samples/README.md b/spring-batch-samples/README.md index a645afa3a..4cbb7764d 100644 --- a/spring-batch-samples/README.md +++ b/spring-batch-samples/README.md @@ -190,6 +190,25 @@ to read and write multiple files in the same step. [MultiResource Input Output Job Sample](./src/main/java/org/springframework/batch/sample/file/multiresource/README.md) +### MultiLine Input Job + +The goal of this sample is to show how to process input files where a single logical +item spans multiple physical line: + +``` +BEGIN +INFO,UK21341EAH45,customer1 +AMNT,978,98.34 +END +BEGIN +INFO,UK21341EAH46,customer2 +AMNT,112,18.12 +END +... +``` + +[MultiLine Input Job Sample](./src/main/java/org/springframework/batch/sample/file/multiline/README.md) + ### Football Job This is a (American) Football statistics loading job. It loads two files containing players and games diff --git a/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/MultiLineJobConfiguration.java b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/MultiLineJobConfiguration.java new file mode 100644 index 000000000..ae1205699 --- /dev/null +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/MultiLineJobConfiguration.java @@ -0,0 +1,84 @@ +package org.springframework.batch.sample.file.multiline; + +import javax.sql.DataSource; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.item.file.FlatFileItemReader; +import org.springframework.batch.item.file.FlatFileItemWriter; +import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder; +import org.springframework.batch.item.file.builder.FlatFileItemWriterBuilder; +import org.springframework.batch.item.file.mapping.PassThroughFieldSetMapper; +import org.springframework.batch.item.file.transform.DelimitedLineTokenizer; +import org.springframework.batch.item.file.transform.FieldSet; +import org.springframework.batch.item.file.transform.PassThroughLineAggregator; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.Resource; +import org.springframework.core.io.WritableResource; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.jdbc.support.JdbcTransactionManager; + +@Configuration +@EnableBatchProcessing +public class MultiLineJobConfiguration { + + @Bean + @StepScope + public MultiLineTradeItemReader itemReader(@Value("#{jobParameters[inputFile]}") Resource resource) { + FlatFileItemReader
delegate = new FlatFileItemReaderBuilder
().name("delegateItemReader") + .resource(resource) + .lineTokenizer(new DelimitedLineTokenizer()) + .fieldSetMapper(new PassThroughFieldSetMapper()) + .build(); + MultiLineTradeItemReader reader = new MultiLineTradeItemReader(); + reader.setDelegate(delegate); + return reader; + } + + @Bean + @StepScope + public MultiLineTradeItemWriter itemWriter(@Value("#{jobParameters[outputFile]}") WritableResource resource) { + FlatFileItemWriter delegate = new FlatFileItemWriterBuilder().name("delegateItemWriter") + .resource(resource) + .lineAggregator(new PassThroughLineAggregator<>()) + .build(); + MultiLineTradeItemWriter writer = new MultiLineTradeItemWriter(); + writer.setDelegate(delegate); + return writer; + } + + @Bean + public Job job(JobRepository jobRepository, JdbcTransactionManager transactionManager, + MultiLineTradeItemReader itemReader, MultiLineTradeItemWriter itemWriter) { + return new JobBuilder("ioSampleJob", jobRepository) + .start(new StepBuilder("step1", jobRepository).chunk(2, transactionManager) + .reader(itemReader) + .writer(itemWriter) + .build()) + .build(); + } + + // Infrastructure beans + + @Bean + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder().setType(EmbeddedDatabaseType.HSQL) + .addScript("/org/springframework/batch/core/schema-drop-hsqldb.sql") + .addScript("/org/springframework/batch/core/schema-hsqldb.sql") + .generateUniqueName(true) + .build(); + } + + @Bean + public JdbcTransactionManager transactionManager(DataSource dataSource) { + return new JdbcTransactionManager(dataSource); + } + +} diff --git a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/internal/MultiLineTradeItemReader.java b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/MultiLineTradeItemReader.java similarity index 95% rename from spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/internal/MultiLineTradeItemReader.java rename to spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/MultiLineTradeItemReader.java index c5888cef6..538dd793c 100644 --- a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/internal/MultiLineTradeItemReader.java +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/MultiLineTradeItemReader.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.batch.sample.iosample.internal; +package org.springframework.batch.sample.file.multiline; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemReader; @@ -22,7 +22,6 @@ import org.springframework.batch.item.ItemStream; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.file.FlatFileItemReader; import org.springframework.batch.item.file.transform.FieldSet; -import org.springframework.batch.sample.domain.trade.Trade; import org.springframework.lang.Nullable; import org.springframework.util.Assert; diff --git a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/internal/MultiLineTradeItemWriter.java b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/MultiLineTradeItemWriter.java similarity index 91% rename from spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/internal/MultiLineTradeItemWriter.java rename to spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/MultiLineTradeItemWriter.java index 18561f630..59d6825e3 100644 --- a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/internal/MultiLineTradeItemWriter.java +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/MultiLineTradeItemWriter.java @@ -1,5 +1,5 @@ /* - * Copyright 2006-2022 the original author or authors. + * Copyright 2006-2023 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.batch.sample.iosample.internal; +package org.springframework.batch.sample.file.multiline; import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; @@ -22,7 +22,6 @@ import org.springframework.batch.item.ItemStream; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemWriter; import org.springframework.batch.item.file.FlatFileItemWriter; -import org.springframework.batch.sample.domain.trade.Trade; /** * @author Dan Garrette diff --git a/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/README.md b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/README.md new file mode 100644 index 000000000..323633e2a --- /dev/null +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/README.md @@ -0,0 +1,31 @@ +### MultiLine Input Job + +## About + +The goal of this sample is to show how to process input files where a single logical +item spans multiple physical line: + +``` +BEGIN +INFO,UK21341EAH45,customer1 +AMNT,978,98.34 +END +BEGIN +INFO,UK21341EAH46,customer2 +AMNT,112,18.12 +END +... +``` + +## Run the sample + +You can run the sample from the command line as following: + +``` +$>cd spring-batch-samples +# Launch the sample using the XML configuration +$>../mvnw -Dtest=MultiLineFunctionalTests#testLaunchJobWithXmlConfig test +# Launch the sample using the Java configuration +$>../mvnw -Dtest=MultiLineFunctionalTests#testLaunchJobWithJavaConfig test +``` + diff --git a/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/Trade.java b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/Trade.java new file mode 100644 index 000000000..f2ff3c1d5 --- /dev/null +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/file/multiline/Trade.java @@ -0,0 +1,169 @@ +/* + * Copyright 2006-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.sample.file.multiline; + +import java.io.Serializable; +import java.math.BigDecimal; + +/** + * @author Rob Harrop + * @author Dave Syer + */ +@SuppressWarnings("serial") +public class Trade implements Serializable { + + private String isin = ""; + + private long quantity = 0; + + private BigDecimal price = BigDecimal.ZERO; + + private String customer = ""; + + private Long id; + + private long version = 0; + + public Trade() { + } + + public Trade(String isin, long quantity, BigDecimal price, String customer) { + this.isin = isin; + this.quantity = quantity; + this.price = price; + this.customer = customer; + } + + /** + * @param id id of the trade + */ + public Trade(long id) { + this.id = id; + } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public long getVersion() { + return version; + } + + public void setVersion(long version) { + this.version = version; + } + + public void setCustomer(String customer) { + this.customer = customer; + } + + public void setIsin(String isin) { + this.isin = isin; + } + + public void setPrice(BigDecimal price) { + this.price = price; + } + + public void setQuantity(long quantity) { + this.quantity = quantity; + } + + public String getIsin() { + return isin; + } + + public BigDecimal getPrice() { + return price; + } + + public long getQuantity() { + return quantity; + } + + public String getCustomer() { + return customer; + } + + @Override + public String toString() { + return "Trade: [isin=" + this.isin + ",quantity=" + this.quantity + ",price=" + this.price + ",customer=" + + this.customer + "]"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((customer == null) ? 0 : customer.hashCode()); + result = prime * result + ((isin == null) ? 0 : isin.hashCode()); + result = prime * result + ((price == null) ? 0 : price.hashCode()); + result = prime * result + (int) (quantity ^ (quantity >>> 32)); + result = prime * result + (int) (version ^ (version >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Trade other = (Trade) obj; + if (customer == null) { + if (other.customer != null) { + return false; + } + } + else if (!customer.equals(other.customer)) { + return false; + } + if (isin == null) { + if (other.isin != null) { + return false; + } + } + else if (!isin.equals(other.isin)) { + return false; + } + if (price == null) { + if (other.price != null) { + return false; + } + } + else if (!price.equals(other.price)) { + return false; + } + if (quantity != other.quantity) { + return false; + } + if (version != other.version) { + return false; + } + return true; + } + +} diff --git a/spring-batch-samples/src/main/resources/data/iosample/input/multiLine.txt b/spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/multiline/data/multiLine.txt similarity index 100% rename from spring-batch-samples/src/main/resources/data/iosample/input/multiLine.txt rename to spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/multiline/data/multiLine.txt diff --git a/spring-batch-samples/src/main/resources/jobs/iosample/multiLine.xml b/spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/multiline/job/multiLine.xml similarity index 80% rename from spring-batch-samples/src/main/resources/jobs/iosample/multiLine.xml rename to spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/multiline/job/multiLine.xml index a26745340..1f5c47fb1 100644 --- a/spring-batch-samples/src/main/resources/jobs/iosample/multiLine.xml +++ b/spring-batch-samples/src/main/resources/org/springframework/batch/sample/file/multiline/job/multiLine.xml @@ -14,10 +14,10 @@ - + - + @@ -32,10 +32,10 @@ - + - + diff --git a/spring-batch-samples/src/test/java/org/springframework/batch/sample/file/multiline/MultiLineFunctionalTests.java b/spring-batch-samples/src/test/java/org/springframework/batch/sample/file/multiline/MultiLineFunctionalTests.java new file mode 100644 index 000000000..1facdb0dc --- /dev/null +++ b/spring-batch-samples/src/test/java/org/springframework/batch/sample/file/multiline/MultiLineFunctionalTests.java @@ -0,0 +1,94 @@ +/* + * Copyright 2006-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.sample.file.multiline; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import org.springframework.batch.core.BatchStatus; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.FileSystemResource; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author Dan Garrette + * @author Mahmoud Ben Hassine + * @author Glenn Renfro + * @since 2.0 + */ +@SpringJUnitConfig(locations = { "/org/springframework/batch/sample/file/multiline/job/multiLine.xml", + "/simple-job-launcher-context.xml", "/job-runner-context.xml" }) +class MultiLineFunctionalTests { + + private static final String INPUT_FILE = "org/springframework/batch/sample/file/multiline/data/multiLine.txt"; + + private static final String OUTPUT_FILE = "target/test-outputs/multiLineOutput.txt"; + + @Autowired + private JobLauncherTestUtils jobLauncherTestUtils; + + @Test + void testLaunchJobWithXmlConfig() throws Exception { + JobParameters jobParameters = new JobParametersBuilder().addString("inputFile", INPUT_FILE) + .addString("outputFile", "file:./" + OUTPUT_FILE) + .toJobParameters(); + + // when + JobExecution jobExecution = this.jobLauncherTestUtils.launchJob(jobParameters); + + // then + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + Path inputFile = new ClassPathResource(INPUT_FILE).getFile().toPath(); + Path outputFile = new FileSystemResource(OUTPUT_FILE).getFile().toPath(); + Assertions.assertLinesMatch(Files.lines(inputFile), Files.lines(outputFile)); + } + + @Test + public void testLaunchJobWithJavaConfig() throws Exception { + // given + ApplicationContext context = new AnnotationConfigApplicationContext(MultiLineJobConfiguration.class); + JobLauncher jobLauncher = context.getBean(JobLauncher.class); + Job job = context.getBean(Job.class); + JobParameters jobParameters = new JobParametersBuilder().addString("inputFile", INPUT_FILE) + .addString("outputFile", "file:./" + OUTPUT_FILE) + .toJobParameters(); + + // when + JobExecution jobExecution = jobLauncher.run(job, jobParameters); + + // then + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + Path inputFile = new ClassPathResource(INPUT_FILE).getFile().toPath(); + Path outputFile = new FileSystemResource(OUTPUT_FILE).getFile().toPath(); + Assertions.assertLinesMatch(Files.lines(inputFile), Files.lines(outputFile)); + } + +} diff --git a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/MultiLineFunctionalTests.java b/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/MultiLineFunctionalTests.java deleted file mode 100644 index 97e3991e8..000000000 --- a/spring-batch-samples/src/test/java/org/springframework/batch/sample/iosample/MultiLineFunctionalTests.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2006-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.batch.sample.iosample; - -import java.nio.file.Files; -import java.nio.file.Path; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.FileSystemResource; -import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; - -/** - * @author Dan Garrette - * @author Mahmoud Ben Hassine - * @author Glenn Renfro - * @since 2.0 - */ -@SpringJUnitConfig( - locations = { "/simple-job-launcher-context.xml", "/jobs/iosample/multiLine.xml", "/job-runner-context.xml" }) -class MultiLineFunctionalTests { - - private static final String OUTPUT_FILE = "target/test-outputs/multiLineOutput.txt"; - - private static final String INPUT_FILE = "src/main/resources/data/iosample/input/multiLine.txt"; - - @Autowired - private JobLauncherTestUtils jobLauncherTestUtils; - - /** - * Output should be the same as input - */ - @Test - void testJob() throws Exception { - // when - this.jobLauncherTestUtils.launchJob(); - - // then - Path inputFile = new FileSystemResource(INPUT_FILE).getFile().toPath(); - Path outputFile = new FileSystemResource(OUTPUT_FILE).getFile().toPath(); - Assertions.assertLinesMatch(Files.lines(inputFile), Files.lines(outputFile)); - } - -}