diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java index 697c67c08..50b7ea787 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java @@ -21,7 +21,8 @@ import java.util.List; /** * Tokenizer used to process data obtained from files with fixed-length format. - * Columns are specified by array of Range objects ({@link #setColumns(Range[])}). + * Columns are specified by array of Range objects ({@link #setColumns(Range[])} + * ). * * @author tomas.slanina * @author peter.zozom @@ -31,45 +32,70 @@ import java.util.List; public class FixedLengthTokenizer extends AbstractLineTokenizer { private Range[] ranges; - private int maxRange; + + private int maxRange = 0; + + boolean open = false; + + private boolean strict = true; + + /** + * Public setter for the strict flag. If true (the default) then lines must + * be precisely the length specified by the columns. If false then shorter + * lines will be tolerated and padded with empty columns, and longer strings + * will simply be truncated. + * + * @see #setColumns(Range[]) + * + * @param strict the strict to set + */ + public void setStrict(boolean strict) { + this.strict = strict; + } /** * Set the column ranges. Used in conjunction with the * {@link RangeArrayPropertyEditor} this property can be set in the form of * a String describing the range boundaries, e.g. "1,4,7" or "1-3,4-6,7" or - * "1-2,4-5,7-10". + * "1-2,4-5,7-10". If the last range is open then the rest of the line is + * read into that column (irrespective of the strict flag setting). + * + * @see #setStrict(boolean) * * @param ranges the column ranges expected in the input */ public void setColumns(Range[] ranges) { this.ranges = ranges; - calculateMaxRange(ranges); } - + /* - * Calculate the highest value within an array of ranges. The ranges aren't - * necessarily in order. For example: "5-10, 1-4,11-15". Furthermore, there + * Calculate the highest value within an array of ranges. The ranges aren't + * necessarily in order. For example: "5-10, 1-4,11-15". Furthermore, there * isn't always a min and max, such as: "1,4-20, 22" */ - private void calculateMaxRange(Range[] ranges){ - if(ranges == null || ranges.length == 0){ + private void calculateMaxRange(Range[] ranges) { + if (ranges == null || ranges.length == 0) { maxRange = 0; return; } - + + open = false; maxRange = ranges[0].getMin(); - - for(int i = 0; i < ranges.length; i++){ + + for (int i = 0; i < ranges.length; i++) { int upperBound; - if(ranges[i].hasMaxValue()){ + if (ranges[i].hasMaxValue()) { upperBound = ranges[i].getMax(); } - else{ + else { upperBound = ranges[i].getMin(); + if (upperBound > maxRange) { + open = true; + } } - - if(upperBound > maxRange){ + + if (upperBound > maxRange) { maxRange = upperBound; } } @@ -79,12 +105,11 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer { * Yields the tokens resulting from the splitting of the supplied * line. * - * @param line - * the line to be tokenised (can be null) + * @param line the line to be tokenised (can be null) * * @return the resulting tokens (empty if the line is null) - * @throws IncorrectLineLengthException if line length is greater than - * or less than the max range set. + * @throws IncorrectLineLengthException if line length is greater than or + * less than the max range set. */ protected List doTokenize(String line) { List tokens = new ArrayList(ranges.length); @@ -92,10 +117,13 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer { String token; lineLength = line.length(); - - if(lineLength > maxRange || lineLength < maxRange){ - //line is longer than max range, throw exception - throw new IncorrectLineLengthException(maxRange, lineLength); + + if (lineLength < maxRange && strict) { + throw new IncorrectLineLengthException("Line is shorter than max range " + maxRange, maxRange, lineLength); + } + + if (!open && lineLength > maxRange && strict) { + throw new IncorrectLineLengthException("Line is longer than max range " + maxRange, maxRange, lineLength); } for (int i = 0; i < ranges.length; i++) { @@ -105,9 +133,11 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer { if (lineLength >= endPos) { token = line.substring(startPos, endPos); - } else if (lineLength >= startPos) { + } + else if (lineLength >= startPos) { token = line.substring(startPos); - } else { + } + else { token = ""; } diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/Range.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/Range.java index 337a7ea34..cd13489dd 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/Range.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/Range.java @@ -14,13 +14,11 @@ public class Range { public final static int UPPER_BORDER_NOT_DEFINED = Integer.MAX_VALUE; - private int min; - private int max; + final private int min; + final private int max; public Range(int min) { - checkMinMaxValues(min, UPPER_BORDER_NOT_DEFINED); - this.min = min; - this.max = UPPER_BORDER_NOT_DEFINED; + this(min,UPPER_BORDER_NOT_DEFINED); } public Range(int min, int max) { @@ -37,16 +35,6 @@ public class Range { return min; } - public void setMax(int max) { - checkMinMaxValues(this.min, max); - this.max = max; - } - - public void setMin(int min) { - checkMinMaxValues(min, this.max); - this.min = min; - } - public boolean hasMaxValue() { return max != UPPER_BORDER_NOT_DEFINED; } diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/RangeArrayPropertyEditor.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/RangeArrayPropertyEditor.java index 864d5e5fc..e39083866 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/RangeArrayPropertyEditor.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/RangeArrayPropertyEditor.java @@ -108,7 +108,7 @@ public class RangeArrayPropertyEditor extends PropertyEditorSupport { for (int i = 0; i < c.length - 1; i++) { if (!c[i].hasMaxValue()) { //set max value to (min value - 1) of the next range - c[i].setMax(c[i+1].getMin() - 1); + c[i] = new Range(c[i].getMin(),c[i+1].getMin() - 1); } } diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java index 6929e3d80..4583c6efb 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java @@ -29,40 +29,40 @@ public class FixedLengthTokenizerTests { private String line = null; /** - * if null or empty string is tokenized, tokenizer returns empty fieldset + * if null or empty string is tokenized, tokenizer returns empty fieldset * (with no tokens). */ @Test public void testTokenizeEmptyString() { - tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)}); - try{ + tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10), new Range(11, 15) }); + try { tokenizer.tokenize(""); fail("Expected IncorrectLineLengthException"); } - catch(IncorrectLineLengthException ex){ + catch (IncorrectLineLengthException ex) { assertEquals(15, ex.getExpectedLength()); assertEquals(0, ex.getActualLength()); } } - + @Test - public void testEmptyStringWithNoRanges(){ - tokenizer.setColumns(new Range[]{}); + public void testEmptyStringWithNoRanges() { + tokenizer.setColumns(new Range[] {}); tokenizer.tokenize(""); } - + @Test public void testTokenizeSmallerStringThanRanges() { - tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)}); - try{ + tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10), new Range(11, 15) }); + try { tokenizer.tokenize("12345"); fail("Expected IncorrectLineLengthException"); } - catch(IncorrectLineLengthException ex){ + catch (IncorrectLineLengthException ex) { assertEquals(15, ex.getExpectedLength()); assertEquals(5, ex.getActualLength()); } - + } @Test @@ -73,19 +73,37 @@ public class FixedLengthTokenizerTests { assertEquals("", tokens.readString(1)); } + @Test + public void testTokenizeSmallerStringThanRangesNotStrict() { + tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10) }); + tokenizer.setStrict(false); + FieldSet tokens = tokenizer.tokenize("12345"); + assertEquals("12345", tokens.readString(0)); + assertEquals("", tokens.readString(1)); + } + + @Test + public void testTokenizeSmallerStringThanRangesWithWhitespaceOpenEnded() { + tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6) }); + FieldSet tokens = tokenizer.tokenize("12345 "); + assertEquals("12345", tokens.readString(0)); + assertEquals("", tokens.readString(1)); + } + @Test public void testTokenizeNullString() { - tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)}); - try{ + tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10), new Range(11, 15) }); + try { tokenizer.tokenize(null); fail("Expected IncorrectLineLengthException"); } - catch(IncorrectLineLengthException ex){} + catch (IncorrectLineLengthException ex) { + } } @Test public void testTokenizeRegularUse() { - tokenizer.setColumns(new Range[] {new Range(1,2),new Range(3,7),new Range(8,12)}); + tokenizer.setColumns(new Range[] { new Range(1, 2), new Range(3, 7), new Range(8, 12) }); // test shorter line as defined by record descriptor line = "H11234512345"; FieldSet tokens = tokenizer.tokenize(line); @@ -94,48 +112,69 @@ public class FixedLengthTokenizerTests { assertEquals("12345", tokens.readString(1)); assertEquals("12345", tokens.readString(2)); } - + @Test public void testNormalLength() throws Exception { - tokenizer.setColumns(new Range[] {new Range(1,10),new Range(11,25),new Range(26,30)}); + tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26, 30) }); // test shorter line as defined by record descriptor line = "H1 12345678 12345"; FieldSet tokens = tokenizer.tokenize(line); assertEquals(3, tokens.getFieldCount()); assertEquals(line.substring(0, 10).trim(), tokens.readString(0)); assertEquals(line.substring(10, 25).trim(), tokens.readString(1)); - assertEquals(line.substring(25).trim(), tokens.readString(2)); + assertEquals(line.substring(25).trim(), tokens.readString(2)); } - + @Test public void testLongerLines() throws Exception { - tokenizer.setColumns(new Range[] {new Range(1,10),new Range(11,25),new Range(26,30)}); + tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26, 30) }); line = "H1 12345678 1234567890"; - try{ + try { tokenizer.tokenize(line); fail("Expected IncorrectLineLengthException"); } - catch(IncorrectLineLengthException ex){ + catch (IncorrectLineLengthException ex) { assertEquals(30, ex.getExpectedLength()); assertEquals(35, ex.getActualLength()); } } + @Test + public void testLongerLinesOpenRange() throws Exception { + tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26) }); + line = "H1 12345678 1234567890"; + FieldSet tokens = tokenizer.tokenize(line); + assertEquals(line.substring(0, 10).trim(), tokens.readString(0)); + assertEquals(line.substring(10, 25).trim(), tokens.readString(1)); + assertEquals(line.substring(25).trim(), tokens.readString(2)); + } + + @Test + public void testLongerLinesNotStrict() throws Exception { + tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26,30) }); + line = "H1 12345678 1234567890"; + tokenizer.setStrict(false); + FieldSet tokens = tokenizer.tokenize(line); + assertEquals(line.substring(0, 10).trim(), tokens.readString(0)); + assertEquals(line.substring(10, 25).trim(), tokens.readString(1)); + assertEquals(line.substring(25, 30).trim(), tokens.readString(2)); + } + @Test public void testNonAdjacentRangesUnsorted() throws Exception { - tokenizer.setColumns(new Range[] {new Range(14,28), new Range(34,38), new Range(1,10)}); + tokenizer.setColumns(new Range[] { new Range(14, 28), new Range(34, 38), new Range(1, 10) }); // test normal length line = "H1 +++12345678 +++++12345"; FieldSet tokens = tokenizer.tokenize(line); assertEquals(3, tokens.getFieldCount()); assertEquals(line.substring(0, 10).trim(), tokens.readString(2)); assertEquals(line.substring(13, 28).trim(), tokens.readString(0)); - assertEquals(line.substring(33, 38).trim(), tokens.readString(1)); + assertEquals(line.substring(33, 38).trim(), tokens.readString(1)); } - + @Test public void testAnotherTypeOfRecord() throws Exception { - tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,15),new Range(16,25),new Range(26,27)}); + tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 15), new Range(16, 25), new Range(26, 27) }); // test another type of record line = "H2 123456 12345 12"; FieldSet tokens = tokenizer.tokenize(line); @@ -143,12 +182,13 @@ public class FixedLengthTokenizerTests { assertEquals(line.substring(0, 5).trim(), tokens.readString(0)); assertEquals(line.substring(5, 15).trim(), tokens.readString(1)); assertEquals(line.substring(15, 25).trim(), tokens.readString(2)); - assertEquals(line.substring(25).trim(), tokens.readString(3)); + assertEquals(line.substring(25).trim(), tokens.readString(3)); } - + @Test public void testFillerAtEnd() throws Exception { - tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,15),new Range(16,25),new Range(26,27),new Range(34)}); + tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 15), new Range(16, 25), new Range(26, 27), + new Range(34) }); // test another type of record line = "H2 123456 12345 12-123456"; FieldSet tokens = tokenizer.tokenize(line); @@ -156,13 +196,13 @@ public class FixedLengthTokenizerTests { assertEquals(line.substring(0, 5).trim(), tokens.readString(0)); assertEquals(line.substring(5, 15).trim(), tokens.readString(1)); assertEquals(line.substring(15, 25).trim(), tokens.readString(2)); - assertEquals(line.substring(25, 27).trim(), tokens.readString(3)); + assertEquals(line.substring(25, 27).trim(), tokens.readString(3)); } @Test public void testTokenizerInvalidSetup() { - tokenizer.setNames(new String[] {"a", "b"}); - tokenizer.setColumns(new Range[] {new Range(1,5)}); + tokenizer.setNames(new String[] { "a", "b" }); + tokenizer.setColumns(new Range[] { new Range(1, 5) }); try { tokenizer.tokenize("12345");