diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/builder/FlatFileItemReaderBuilder.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/builder/FlatFileItemReaderBuilder.java new file mode 100644 index 000000000..b7f3bb7f7 --- /dev/null +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/builder/FlatFileItemReaderBuilder.java @@ -0,0 +1,354 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.item.file.builder; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import org.springframework.batch.item.file.FlatFileItemReader; +import org.springframework.batch.item.file.LineCallbackHandler; +import org.springframework.batch.item.file.LineMapper; +import org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper; +import org.springframework.batch.item.file.mapping.DefaultLineMapper; +import org.springframework.batch.item.file.mapping.FieldSetMapper; +import org.springframework.batch.item.file.separator.RecordSeparatorPolicy; +import org.springframework.batch.item.file.separator.SimpleRecordSeparatorPolicy; +import org.springframework.batch.item.file.transform.DefaultFieldSetFactory; +import org.springframework.batch.item.file.transform.DelimitedLineTokenizer; +import org.springframework.batch.item.file.transform.FieldSetFactory; +import org.springframework.batch.item.file.transform.FixedLengthTokenizer; +import org.springframework.batch.item.file.transform.LineTokenizer; +import org.springframework.batch.item.file.transform.Range; +import org.springframework.core.io.Resource; +import org.springframework.util.StringUtils; + +/** + * @author Michael Minella + */ +public class FlatFileItemReaderBuilder { + + private String name; + + private boolean strict = true; + + private RecordSeparatorPolicy recordSeparatorPolicy = + new SimpleRecordSeparatorPolicy(); + + private Resource resource; + + private int maxItemCount = Integer.MAX_VALUE; + + private List comments = new ArrayList<>(); + + private int linesToSkip = 0; + + private LineCallbackHandler skippedLinesCallback; + + private LineMapper lineMapper; + + private FieldSetMapper fieldSetMapper; + + private LineTokenizer lineTokenizer; + + private DelimitedBuilder delimitedBuilder; + + private FixedLengthBuilder fixedLengthBuilder; + + private Class beanMapperClass; + + private boolean saveState = true; + + public FlatFileItemReaderBuilder strict(boolean strict) { + this.strict = strict; + return this; + } + + public FlatFileItemReaderBuilder recordSeparatorPolicy(RecordSeparatorPolicy policy) { + this.recordSeparatorPolicy = policy; + return this; + } + + public FlatFileItemReaderBuilder resource(Resource resource) { + this.resource = resource; + return this; + } + + public FlatFileItemReaderBuilder maxItemCount(int maxItemCount) { + this.maxItemCount = maxItemCount; + return this; + } + + public FlatFileItemReaderBuilder comments(String[] comments) { + this.comments.addAll(Arrays.asList(comments)); + return this; + } + + public FlatFileItemReaderBuilder addComment(String comment) { + this.comments.add(comment); + return this; + } + + public FlatFileItemReaderBuilder linesToSkip(int linesToSkip) { + this.linesToSkip = linesToSkip; + return this; + } + + public FlatFileItemReaderBuilder skippedLinesCallback(LineCallbackHandler callback) { + this.skippedLinesCallback = callback; + return this; + } + + public FlatFileItemReaderBuilder lineMapper(LineMapper lineMapper) { + this.lineMapper = lineMapper; + return this; + } + + public FlatFileItemReaderBuilder fieldSetMapper(FieldSetMapper mapper) { + this.fieldSetMapper = mapper; + return this; + } + + public FlatFileItemReaderBuilder lineTokenizer(LineTokenizer tokenizer) { + this.lineTokenizer = tokenizer; + return this; + } + + public DelimitedBuilder delimited() { + this.delimitedBuilder = new DelimitedBuilder<>(this); + return this.delimitedBuilder; + } + + public FixedLengthBuilder fixedLength() { + this.fixedLengthBuilder = new FixedLengthBuilder<>(this); + return this.fixedLengthBuilder; + } + + public FlatFileItemReaderBuilder beanMapperClass(Class beanMapperClass) { + this.beanMapperClass = beanMapperClass; + return this; + } + + public FlatFileItemReaderBuilder saveState(boolean saveState) { + this.saveState = saveState; + return this; + } + + public FlatFileItemReaderBuilder name(String name) { + this.name = name; + return this; + } + + public FlatFileItemReader build() throws Exception { + FlatFileItemReader reader = new FlatFileItemReader<>(); + + reader.setName(this.name); + reader.setResource(this.resource); + + if(this.lineMapper != null) { + reader.setLineMapper(this.lineMapper); + } + else { + DefaultLineMapper lineMapper = new DefaultLineMapper<>(); + + if(this.lineTokenizer != null) { + lineMapper.setLineTokenizer(this.lineTokenizer); + lineMapper.setFieldSetMapper(this.fieldSetMapper); + } + else if(this.fixedLengthBuilder != null) { + lineMapper.setLineTokenizer(this.fixedLengthBuilder.build()); + } + else if(this.delimitedBuilder != null) { + lineMapper.setLineTokenizer(this.delimitedBuilder.build()); + } + else { + throw new IllegalStateException("No LineTokenizer implementation was provided"); + } + + if(this.beanMapperClass != null) { + BeanWrapperFieldSetMapper mapper = new BeanWrapperFieldSetMapper(); + mapper.setTargetType(this.beanMapperClass); + mapper.afterPropertiesSet(); + + lineMapper.setFieldSetMapper(mapper); + } + + reader.setLineMapper(lineMapper); + } + + reader.setLinesToSkip(this.linesToSkip); + reader.setComments(this.comments.toArray(new String[this.comments.size()])); + reader.setSkippedLinesCallback(this.skippedLinesCallback); + reader.setRecordSeparatorPolicy(this.recordSeparatorPolicy); + reader.setMaxItemCount(this.maxItemCount); + reader.setSaveState(this.saveState); + reader.setStrict(this.strict); + + return reader; + } + + public static class DelimitedBuilder { + private FlatFileItemReaderBuilder parent; + + private List names = new ArrayList<>(); + + private String delimiter; + + private Character quoteCharacter; + + private List includedFields = new ArrayList<>(); + + private FieldSetFactory fieldSetFactory = new DefaultFieldSetFactory(); + + private boolean strict = true; + + public DelimitedBuilder(FlatFileItemReaderBuilder parent) { + this.parent = parent; + } + + public DelimitedBuilder delimiter(String delimiter) { + this.delimiter = delimiter; + return this; + } + + public DelimitedBuilder quoteCharacter(char quoteCharacter) { + this.quoteCharacter = quoteCharacter; + return this; + } + + public DelimitedBuilder includedFields(Integer[] fields) { + this.includedFields.addAll(Arrays.asList(fields)); + return this; + } + + public DelimitedBuilder addIncludedField(int field) { + this.includedFields.add(field); + return this; + } + + public DelimitedBuilder fieldSetFactory(FieldSetFactory fieldSetFactory) { + this.fieldSetFactory = fieldSetFactory; + return this; + } + + public FlatFileItemReaderBuilder names(String [] names) { + this.names.addAll(Arrays.asList(names)); + return this.parent; + } + + public DelimitedLineTokenizer build() throws Exception { + DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer(); + + tokenizer.setNames(this.names.toArray(new String[this.names.size()])); + + if(StringUtils.hasText(this.delimiter)) { + tokenizer.setDelimiter(this.delimiter); + } + + if(this.quoteCharacter != null) { + tokenizer.setQuoteCharacter(this.quoteCharacter); + } + + if(!this.includedFields.isEmpty()) { + Set deDupedFields = new HashSet<>(this.includedFields.size()); + deDupedFields.addAll(this.includedFields); + deDupedFields.remove(null); + + int [] fields = new int[deDupedFields.size()]; + Iterator iterator = deDupedFields.iterator(); + for(int i = 0; i < fields.length; i++) { + fields[i] = iterator.next(); + } + + tokenizer.setIncludedFields(fields); + } + + tokenizer.setFieldSetFactory(this.fieldSetFactory); + tokenizer.setStrict(this.strict); + + tokenizer.afterPropertiesSet(); + + return tokenizer; + } + } + + public static class FixedLengthBuilder { + private FlatFileItemReaderBuilder parent; + + private List ranges = new ArrayList<>(); + + private int maxRange = 0; + + private List names = new ArrayList<>(); + + private boolean strict = true; + + private FieldSetFactory fieldSetFactory = new DefaultFieldSetFactory(); + + public FixedLengthBuilder(FlatFileItemReaderBuilder parent) { + this.parent = parent; + } + + public FixedLengthBuilder ranges(Range[] ranges) { + this.ranges.addAll(Arrays.asList(ranges)); + return this; + } + + public FixedLengthBuilder addRange(Range range) { + this.ranges.add(range); + return this; + } + + public FixedLengthBuilder addRange(Range range, int index) { + this.ranges.add(index, range); + return this; + } + + public FixedLengthBuilder maxRange(int maxRange) { + this.maxRange = maxRange; + return this; + } + + public FlatFileItemReaderBuilder names(String [] names) { + this.names.addAll(Arrays.asList(names)); + return this.parent; + } + + public FixedLengthBuilder strict(boolean strict) { + this.strict = strict; + return this; + } + + public FixedLengthBuilder fieldSetFactory(FieldSetFactory factory) { + this.fieldSetFactory = factory; + return this; + } + + public FixedLengthTokenizer build() { + FixedLengthTokenizer tokenizer = new FixedLengthTokenizer(); + + tokenizer.setNames(this.names.toArray(new String[this.names.size()])); + tokenizer.setColumns(this.ranges.toArray(new Range[this.ranges.size()])); + tokenizer.setFieldSetFactory(this.fieldSetFactory); + tokenizer.setStrict(this.strict); + + return tokenizer; + } + } +} diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/builder/FlatFileItemReaderBuilderTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/builder/FlatFileItemReaderBuilderTests.java new file mode 100644 index 000000000..9e95b5fd7 --- /dev/null +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/builder/FlatFileItemReaderBuilderTests.java @@ -0,0 +1,232 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.item.file.builder; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.item.file.FlatFileItemReader; +import org.springframework.batch.item.file.separator.DefaultRecordSeparatorPolicy; +import org.springframework.batch.item.file.transform.DefaultFieldSet; +import org.springframework.batch.item.file.transform.Range; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * @author Michael Minella + */ +public class FlatFileItemReaderBuilderTests { + + @Test + public void testSimpleFixedLength() throws Exception { + FlatFileItemReader reader = new FlatFileItemReaderBuilder() + .name("fooReader") + .resource(getResource("1 2 3")) + .fixedLength() + .ranges(new Range[] {new Range(1, 3), new Range(4, 6), new Range(7)}) + .names(new String[] {"first", "second", "third"}) + .beanMapperClass(Foo.class) + .build(); + + reader.open(new ExecutionContext()); + Foo item = reader.read(); + assertEquals(1, item.getFirst()); + assertEquals(2, item.getSecond()); + assertEquals("3", item.getThird()); + assertNull(reader.read()); + } + + @Test + public void testSimpleDelimited() throws Exception { + FlatFileItemReader reader = new FlatFileItemReaderBuilder() + .name("fooReader") + .resource(getResource("1,2,3")) + .delimited() + .names(new String[] {"first", "second", "third"}) + .beanMapperClass(Foo.class) + .build(); + + reader.open(new ExecutionContext()); + Foo item = reader.read(); + assertEquals(1, item.getFirst()); + assertEquals(2, item.getSecond()); + assertEquals("3", item.getThird()); + assertNull(reader.read()); + } + + @Test + public void testAdvancedDelimited() throws Exception { + final List skippedLines = new ArrayList<>(); + + FlatFileItemReader reader = new FlatFileItemReaderBuilder() + .name("fooReader") + .resource(getResource("1,2,3\n4,5,$1,2,3$\n@this is a comment\n6,7, 8")) + .delimited() + .quoteCharacter('$') + .names(new String[] {"first", "second", "third"}) + .beanMapperClass(Foo.class) + .linesToSkip(1) + .skippedLinesCallback(skippedLines::add) + .addComment("@") + .build(); + + ExecutionContext executionContext = new ExecutionContext(); + reader.open(executionContext); + + Foo item = reader.read(); + assertEquals(4, item.getFirst()); + assertEquals(5, item.getSecond()); + assertEquals("1,2,3", item.getThird()); + + item = reader.read(); + assertEquals(6, item.getFirst()); + assertEquals(7, item.getSecond()); + assertEquals("8", item.getThird()); + + reader.update(executionContext); + + assertNull(reader.read()); + + assertEquals("1,2,3", skippedLines.get(0)); + assertEquals(1, skippedLines.size()); + + assertEquals(1, executionContext.size()); + } + + @Test + public void testAdvancedFixedLength() throws Exception { + FlatFileItemReader reader = new FlatFileItemReaderBuilder() + .name("fooReader") + .resource(getResource("1 2%\n 3\n4 5%\n 6\n@this is a comment\n7 8%\n 9\n")) + .fixedLength() + .ranges(new Range[] {new Range(1, 2), new Range(3, 5), new Range(6)}) + .names(new String[] {"first", "second", "third"}) + .beanMapperClass(Foo.class) + .recordSeparatorPolicy(new DefaultRecordSeparatorPolicy("\"", "%")) + .maxItemCount(2) + .saveState(false) + .build(); + + ExecutionContext executionContext = new ExecutionContext(); + + reader.open(executionContext); + Foo item = reader.read(); + assertEquals(1, item.getFirst()); + assertEquals(2, item.getSecond()); + assertEquals("3", item.getThird()); + + item = reader.read(); + assertEquals(4, item.getFirst()); + assertEquals(5, item.getSecond()); + assertEquals("6", item.getThird()); + + reader.update(executionContext); + + assertNull(reader.read()); + assertEquals(0, executionContext.size()); + } + + @Test + public void testStrict() throws Exception { + FlatFileItemReader reader = new FlatFileItemReaderBuilder() + .name("fooReader") + .resource(new FileSystemResource("this/file/does/not/exist")) + .delimited() + .names(new String[] {"first", "second", "third"}) + .beanMapperClass(Foo.class) + .strict(false) + .build(); + + reader.open(new ExecutionContext()); + + assertNull(reader.read()); + } + + @Test + public void testCustomLineTokenizerFieldSetMapper() throws Exception { + FlatFileItemReader reader = new FlatFileItemReaderBuilder() + .name("fooReader") + .resource(getResource("|1|&|2|&| 3|\n|4|&|5|&|foo|")) + .lineTokenizer(line -> new DefaultFieldSet(line.split("&"))) + .fieldSetMapper(fieldSet -> { + Foo item = new Foo(); + + item.setFirst(Integer.valueOf(fieldSet.readString(0).replaceAll("\\|", ""))); + item.setSecond(Integer.valueOf(fieldSet.readString(1).replaceAll("\\|", ""))); + item.setThird(fieldSet.readString(2).replaceAll("\\|", "")); + + return item; + }) + .build(); + + reader.open(new ExecutionContext()); + Foo item = reader.read(); + + assertEquals(1, item.getFirst()); + assertEquals(2, item.getSecond()); + assertEquals(" 3", item.getThird()); + + item = reader.read(); + + assertEquals(4, item.getFirst()); + assertEquals(5, item.getSecond()); + assertEquals("foo", item.getThird()); + + assertNull(reader.read()); + } + + private Resource getResource(String contents) { + return new ByteArrayResource(contents.getBytes()); + } + + public static class Foo { + private int first; + private int second; + private String third; + + public int getFirst() { + return first; + } + + public void setFirst(int first) { + this.first = first; + } + + public int getSecond() { + return second; + } + + public void setSecond(int second) { + this.second = second; + } + + public String getThird() { + return third; + } + + public void setThird(String third) { + this.third = third; + } + } + +}