From ea1f61ae19a897510dbfdf4d1d12f69dbe4ecf0c Mon Sep 17 00:00:00 2001 From: dsyer Date: Thu, 4 Sep 2008 08:16:09 +0000 Subject: [PATCH] RESOLVED - issue BATCH-809: fixedLenghtTokenizer problem with 1.1.2 Add strict flag and open-ended last range logic --- .../file/transform/FixedLengthTokenizer.java | 82 +++++++++++++------ .../batch/item/file/transform/Range.java | 18 +--- .../transform/RangeArrayPropertyEditor.java | 2 +- .../transform/FixedLengthTokenizerTests.java | 34 ++++++++ 4 files changed, 94 insertions(+), 42 deletions(-) diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java index 86254cd8b..edbd2d9d8 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java @@ -21,7 +21,8 @@ import java.util.List; /** * Tokenizer used to process data obtained from files with fixed-length format. - * Columns are specified by array of Range objects ({@link #setColumns(Range[])}). + * Columns are specified by array of Range objects ({@link #setColumns(Range[])} + * ). * * @author tomas.slanina * @author peter.zozom @@ -31,45 +32,70 @@ import java.util.List; public class FixedLengthTokenizer extends AbstractLineTokenizer { private Range[] ranges; - private int maxRange; + + private int maxRange = 0; + + boolean open = false; + + private boolean strict = true; + + /** + * Public setter for the strict flag. If true (the default) then lines must + * be precisely the length specified by the columns. If false then shorter + * lines will be tolerated and padded with empty columns, and longer strings + * will simply be truncated. + * + * @see #setColumns(Range[]) + * + * @param strict the strict to set + */ + public void setStrict(boolean strict) { + this.strict = strict; + } /** * Set the column ranges. Used in conjunction with the * {@link RangeArrayPropertyEditor} this property can be set in the form of * a String describing the range boundaries, e.g. "1,4,7" or "1-3,4-6,7" or - * "1-2,4-5,7-10". + * "1-2,4-5,7-10". If the last range is open then the rest of the line is + * read into that column (irrespective of the strict flag setting). + * + * @see #setStrict(boolean) * * @param ranges the column ranges expected in the input */ public void setColumns(Range[] ranges) { this.ranges = ranges; - calculateMaxRange(ranges); } - + /* - * Calculate the highest value within an array of ranges. The ranges aren't - * necessarily in order. For example: "5-10, 1-4,11-15". Furthermore, there + * Calculate the highest value within an array of ranges. The ranges aren't + * necessarily in order. For example: "5-10, 1-4,11-15". Furthermore, there * isn't always a min and max, such as: "1,4-20, 22" */ - private void calculateMaxRange(Range[] ranges){ - if(ranges == null || ranges.length == 0){ + private void calculateMaxRange(Range[] ranges) { + if (ranges == null || ranges.length == 0) { maxRange = 0; return; } - + + open = false; maxRange = ranges[0].getMin(); - - for(int i = 0; i < ranges.length; i++){ + + for (int i = 0; i < ranges.length; i++) { int upperBound; - if(ranges[i].hasMaxValue()){ + if (ranges[i].hasMaxValue()) { upperBound = ranges[i].getMax(); } - else{ + else { upperBound = ranges[i].getMin(); + if (upperBound > maxRange) { + open = true; + } } - - if(upperBound > maxRange){ + + if (upperBound > maxRange) { maxRange = upperBound; } } @@ -79,12 +105,11 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer { * Yields the tokens resulting from the splitting of the supplied * line. * - * @param line - * the line to be tokenised (can be null) + * @param line the line to be tokenised (can be null) * * @return the resulting tokens (empty if the line is null) - * @throws IncorrectLineLengthException if line length is greater than - * or less than the max range set. + * @throws IncorrectLineLengthException if line length is greater than or + * less than the max range set. */ protected List doTokenize(String line) { List tokens = new ArrayList(ranges.length); @@ -92,10 +117,13 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer { String token; lineLength = line.length(); - - if(lineLength > maxRange || lineLength < maxRange){ - //line is longer than max range, throw exception - throw new IncorrectLineLengthException(maxRange, lineLength); + + if (lineLength < maxRange && strict) { + throw new IncorrectLineLengthException("Line is shorter than max range " + maxRange, maxRange, lineLength); + } + + if (!open && lineLength > maxRange && strict) { + throw new IncorrectLineLengthException("Line is longer than max range " + maxRange, maxRange, lineLength); } for (int i = 0; i < ranges.length; i++) { @@ -105,9 +133,11 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer { if (lineLength >= endPos) { token = line.substring(startPos, endPos); - } else if (lineLength >= startPos) { + } + else if (lineLength >= startPos) { token = line.substring(startPos); - } else { + } + else { token = ""; } diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/Range.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/Range.java index 337a7ea34..cd13489dd 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/Range.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/Range.java @@ -14,13 +14,11 @@ public class Range { public final static int UPPER_BORDER_NOT_DEFINED = Integer.MAX_VALUE; - private int min; - private int max; + final private int min; + final private int max; public Range(int min) { - checkMinMaxValues(min, UPPER_BORDER_NOT_DEFINED); - this.min = min; - this.max = UPPER_BORDER_NOT_DEFINED; + this(min,UPPER_BORDER_NOT_DEFINED); } public Range(int min, int max) { @@ -37,16 +35,6 @@ public class Range { return min; } - public void setMax(int max) { - checkMinMaxValues(this.min, max); - this.max = max; - } - - public void setMin(int min) { - checkMinMaxValues(min, this.max); - this.min = min; - } - public boolean hasMaxValue() { return max != UPPER_BORDER_NOT_DEFINED; } diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/RangeArrayPropertyEditor.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/RangeArrayPropertyEditor.java index 568bd8779..2b5fa5322 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/RangeArrayPropertyEditor.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/RangeArrayPropertyEditor.java @@ -110,7 +110,7 @@ public class RangeArrayPropertyEditor extends PropertyEditorSupport { for (int i = 0; i < c.length - 1; i++) { if (!c[i].hasMaxValue()) { //set max value to (min value - 1) of the next range - c[i].setMax(c[i+1].getMin() - 1); + c[i] = new Range(c[i].getMin(),c[i+1].getMin() - 1); } } diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java index 2fffce3bf..0a415b659 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java @@ -66,6 +66,21 @@ public class FixedLengthTokenizerTests extends TestCase { assertEquals("", tokens.readString(1)); } + public void testTokenizeSmallerStringThanRangesNotStrict() { + tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10) }); + tokenizer.setStrict(false); + FieldSet tokens = tokenizer.tokenize("12345"); + assertEquals("12345", tokens.readString(0)); + assertEquals("", tokens.readString(1)); + } + + public void testTokenizeSmallerStringThanRangesWithWhitespaceOpenEnded() { + tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6) }); + FieldSet tokens = tokenizer.tokenize("12345 "); + assertEquals("12345", tokens.readString(0)); + assertEquals("", tokens.readString(1)); + } + public void testTokenizeNullString() { tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10), new Range(11, 15) }); @@ -113,6 +128,25 @@ public class FixedLengthTokenizerTests extends TestCase { } } + public void testLongerLinesOpenRange() throws Exception { + tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26) }); + line = "H1 12345678 1234567890"; + FieldSet tokens = tokenizer.tokenize(line); + assertEquals(line.substring(0, 10).trim(), tokens.readString(0)); + assertEquals(line.substring(10, 25).trim(), tokens.readString(1)); + assertEquals(line.substring(25).trim(), tokens.readString(2)); + } + + public void testLongerLinesNotStrict() throws Exception { + tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26,30) }); + line = "H1 12345678 1234567890"; + tokenizer.setStrict(false); + FieldSet tokens = tokenizer.tokenize(line); + assertEquals(line.substring(0, 10).trim(), tokens.readString(0)); + assertEquals(line.substring(10, 25).trim(), tokens.readString(1)); + assertEquals(line.substring(25, 30).trim(), tokens.readString(2)); + } + public void testNonAdjacentRangesUnsorted() throws Exception { tokenizer.setColumns(new Range[] { new Range(14, 28), new Range(34, 38), new Range(1, 10) });