Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

OPEN - issue BATCH-809: fixedLenghtTokenizer problem with 1.1.2

Browse Source 
Add strict flag and open-ended last range logic
This commit is contained in:
dsyer
2008-09-04 08:15:47 +00:00
parent 03aa49f09d
commit 4bc943537d
4 changed files with 133 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java
View File

@@ -21,7 +21,8 @@ import java.util.List;
/**
* Tokenizer used to process data obtained from files with fixed-length format.
* Columns are specified by array of Range objects ({@link #setColumns(Range[])}).
* Columns are specified by array of Range objects ({@link #setColumns(Range[])}
* ).
*
* @author tomas.slanina
* @author peter.zozom
@@ -31,45 +32,70 @@ import java.util.List;
public class FixedLengthTokenizer extends AbstractLineTokenizer {
private Range[] ranges;
private int maxRange;
private int maxRange = 0;
boolean open = false;
private boolean strict = true;
/**
* Public setter for the strict flag. If true (the default) then lines must
* be precisely the length specified by the columns. If false then shorter
* lines will be tolerated and padded with empty columns, and longer strings
* will simply be truncated.
*
* @see #setColumns(Range[])
*
* @param strict the strict to set
*/
public void setStrict(boolean strict) {
this.strict = strict;
}
/**
* Set the column ranges. Used in conjunction with the
* {@link RangeArrayPropertyEditor} this property can be set in the form of
* a String describing the range boundaries, e.g. "1,4,7" or "1-3,4-6,7" or
* "1-2,4-5,7-10".
* "1-2,4-5,7-10". If the last range is open then the rest of the line is
* read into that column (irrespective of the strict flag setting).
*
* @see #setStrict(boolean)
*
* @param ranges the column ranges expected in the input
*/
public void setColumns(Range[] ranges) {
this.ranges = ranges;
calculateMaxRange(ranges);
}
/*
* Calculate the highest value within an array of ranges. The ranges aren't
* necessarily in order. For example: "5-10, 1-4,11-15". Furthermore, there
* Calculate the highest value within an array of ranges. The ranges aren't
* necessarily in order. For example: "5-10, 1-4,11-15". Furthermore, there
* isn't always a min and max, such as: "1,4-20, 22"
*/
private void calculateMaxRange(Range[] ranges){
if(ranges == null || ranges.length == 0){
private void calculateMaxRange(Range[] ranges) {
if (ranges == null || ranges.length == 0) {
maxRange = 0;
return;
}
open = false;
maxRange = ranges[0].getMin();
for(int i = 0; i < ranges.length; i++){
for (int i = 0; i < ranges.length; i++) {
int upperBound;
if(ranges[i].hasMaxValue()){
if (ranges[i].hasMaxValue()) {
upperBound = ranges[i].getMax();
}
else{
else {
upperBound = ranges[i].getMin();
if (upperBound > maxRange) {
open = true;
}
}
if(upperBound > maxRange){
if (upperBound > maxRange) {
maxRange = upperBound;
}
}
@@ -79,12 +105,11 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer {
* Yields the tokens resulting from the splitting of the supplied
* <code>line</code>.
*
* @param line
* the line to be tokenised (can be <code>null</code>)
* @param line the line to be tokenised (can be <code>null</code>)
*
* @return the resulting tokens (empty if the line is null)
* @throws IncorrectLineLengthException if line length is greater than
* or less than the max range set.
* @throws IncorrectLineLengthException if line length is greater than or
* less than the max range set.
*/
protected List<String> doTokenize(String line) {
List<String> tokens = new ArrayList<String>(ranges.length);
@@ -92,10 +117,13 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer {
String token;
lineLength = line.length();
if(lineLength > maxRange || lineLength < maxRange){
//line is longer than max range, throw exception
throw new IncorrectLineLengthException(maxRange, lineLength);
if (lineLength < maxRange && strict) {
throw new IncorrectLineLengthException("Line is shorter than max range " + maxRange, maxRange, lineLength);
}
if (!open && lineLength > maxRange && strict) {
throw new IncorrectLineLengthException("Line is longer than max range " + maxRange, maxRange, lineLength);
}
for (int i = 0; i < ranges.length; i++) {
@@ -105,9 +133,11 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer {
if (lineLength >= endPos) {
token = line.substring(startPos, endPos);
} else if (lineLength >= startPos) {
}
else if (lineLength >= startPos) {
token = line.substring(startPos);
} else {
}
else {
token = "";
}

18
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/Range.java
View File

@@ -14,13 +14,11 @@ public class Range {
public final static int UPPER_BORDER_NOT_DEFINED = Integer.MAX_VALUE;
private int min;
private int max;
final private int min;
final private int max;
public Range(int min) {
checkMinMaxValues(min, UPPER_BORDER_NOT_DEFINED);
this.min = min;
this.max = UPPER_BORDER_NOT_DEFINED;
this(min,UPPER_BORDER_NOT_DEFINED);
}
public Range(int min, int max) {
@@ -37,16 +35,6 @@ public class Range {
return min;
}
public void setMax(int max) {
checkMinMaxValues(this.min, max);
this.max = max;
}
public void setMin(int min) {
checkMinMaxValues(min, this.max);
this.min = min;
}
public boolean hasMaxValue() {
return max != UPPER_BORDER_NOT_DEFINED;
}

2
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/RangeArrayPropertyEditor.java
View File

@@ -108,7 +108,7 @@ public class RangeArrayPropertyEditor extends PropertyEditorSupport {
for (int i = 0; i < c.length - 1; i++) {
if (!c[i].hasMaxValue()) {
//set max value to (min value - 1) of the next range
c[i].setMax(c[i+1].getMin() - 1);
c[i] = new Range(c[i].getMin(),c[i+1].getMin() - 1);
}
}

106
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java
View File

@@ -29,40 +29,40 @@ public class FixedLengthTokenizerTests {
private String line = null;
/**
* if null or empty string is tokenized, tokenizer returns empty fieldset
* if null or empty string is tokenized, tokenizer returns empty fieldset
* (with no tokens).
*/
@Test
public void testTokenizeEmptyString() {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)});
try{
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10), new Range(11, 15) });
try {
tokenizer.tokenize("");
fail("Expected IncorrectLineLengthException");
}
catch(IncorrectLineLengthException ex){
catch (IncorrectLineLengthException ex) {
assertEquals(15, ex.getExpectedLength());
assertEquals(0, ex.getActualLength());
}
}
@Test
public void testEmptyStringWithNoRanges(){
tokenizer.setColumns(new Range[]{});
public void testEmptyStringWithNoRanges() {
tokenizer.setColumns(new Range[] {});
tokenizer.tokenize("");
}
@Test
public void testTokenizeSmallerStringThanRanges() {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)});
try{
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10), new Range(11, 15) });
try {
tokenizer.tokenize("12345");
fail("Expected IncorrectLineLengthException");
}
catch(IncorrectLineLengthException ex){
catch (IncorrectLineLengthException ex) {
assertEquals(15, ex.getExpectedLength());
assertEquals(5, ex.getActualLength());
}
}
@Test
@@ -73,19 +73,37 @@ public class FixedLengthTokenizerTests {
assertEquals("", tokens.readString(1));
}
@Test
public void testTokenizeSmallerStringThanRangesNotStrict() {
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10) });
tokenizer.setStrict(false);
FieldSet tokens = tokenizer.tokenize("12345");
assertEquals("12345", tokens.readString(0));
assertEquals("", tokens.readString(1));
}
@Test
public void testTokenizeSmallerStringThanRangesWithWhitespaceOpenEnded() {
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6) });
FieldSet tokens = tokenizer.tokenize("12345 ");
assertEquals("12345", tokens.readString(0));
assertEquals("", tokens.readString(1));
}
@Test
public void testTokenizeNullString() {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)});
try{
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10), new Range(11, 15) });
try {
tokenizer.tokenize(null);
fail("Expected IncorrectLineLengthException");
}
catch(IncorrectLineLengthException ex){}
catch (IncorrectLineLengthException ex) {
}
}
@Test
public void testTokenizeRegularUse() {
tokenizer.setColumns(new Range[] {new Range(1,2),new Range(3,7),new Range(8,12)});
tokenizer.setColumns(new Range[] { new Range(1, 2), new Range(3, 7), new Range(8, 12) });
// test shorter line as defined by record descriptor
line = "H11234512345";
FieldSet tokens = tokenizer.tokenize(line);
@@ -94,48 +112,69 @@ public class FixedLengthTokenizerTests {
assertEquals("12345", tokens.readString(1));
assertEquals("12345", tokens.readString(2));
}
@Test
public void testNormalLength() throws Exception {
tokenizer.setColumns(new Range[] {new Range(1,10),new Range(11,25),new Range(26,30)});
tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26, 30) });
// test shorter line as defined by record descriptor
line = "H1 12345678 12345";
FieldSet tokens = tokenizer.tokenize(line);
assertEquals(3, tokens.getFieldCount());
assertEquals(line.substring(0, 10).trim(), tokens.readString(0));
assertEquals(line.substring(10, 25).trim(), tokens.readString(1));
assertEquals(line.substring(25).trim(), tokens.readString(2));
assertEquals(line.substring(25).trim(), tokens.readString(2));
}
@Test
public void testLongerLines() throws Exception {
tokenizer.setColumns(new Range[] {new Range(1,10),new Range(11,25),new Range(26,30)});
tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26, 30) });
line = "H1 12345678 1234567890";
try{
try {
tokenizer.tokenize(line);
fail("Expected IncorrectLineLengthException");
}
catch(IncorrectLineLengthException ex){
catch (IncorrectLineLengthException ex) {
assertEquals(30, ex.getExpectedLength());
assertEquals(35, ex.getActualLength());
}
}
@Test
public void testLongerLinesOpenRange() throws Exception {
tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26) });
line = "H1 12345678 1234567890";
FieldSet tokens = tokenizer.tokenize(line);
assertEquals(line.substring(0, 10).trim(), tokens.readString(0));
assertEquals(line.substring(10, 25).trim(), tokens.readString(1));
assertEquals(line.substring(25).trim(), tokens.readString(2));
}
@Test
public void testLongerLinesNotStrict() throws Exception {
tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26,30) });
line = "H1 12345678 1234567890";
tokenizer.setStrict(false);
FieldSet tokens = tokenizer.tokenize(line);
assertEquals(line.substring(0, 10).trim(), tokens.readString(0));
assertEquals(line.substring(10, 25).trim(), tokens.readString(1));
assertEquals(line.substring(25, 30).trim(), tokens.readString(2));
}
@Test
public void testNonAdjacentRangesUnsorted() throws Exception {
tokenizer.setColumns(new Range[] {new Range(14,28), new Range(34,38), new Range(1,10)});
tokenizer.setColumns(new Range[] { new Range(14, 28), new Range(34, 38), new Range(1, 10) });
// test normal length
line = "H1 +++12345678 +++++12345";
FieldSet tokens = tokenizer.tokenize(line);
assertEquals(3, tokens.getFieldCount());
assertEquals(line.substring(0, 10).trim(), tokens.readString(2));
assertEquals(line.substring(13, 28).trim(), tokens.readString(0));
assertEquals(line.substring(33, 38).trim(), tokens.readString(1));
assertEquals(line.substring(33, 38).trim(), tokens.readString(1));
}
@Test
public void testAnotherTypeOfRecord() throws Exception {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,15),new Range(16,25),new Range(26,27)});
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 15), new Range(16, 25), new Range(26, 27) });
// test another type of record
line = "H2 123456 12345 12";
FieldSet tokens = tokenizer.tokenize(line);
@@ -143,12 +182,13 @@ public class FixedLengthTokenizerTests {
assertEquals(line.substring(0, 5).trim(), tokens.readString(0));
assertEquals(line.substring(5, 15).trim(), tokens.readString(1));
assertEquals(line.substring(15, 25).trim(), tokens.readString(2));
assertEquals(line.substring(25).trim(), tokens.readString(3));
assertEquals(line.substring(25).trim(), tokens.readString(3));
}
@Test
public void testFillerAtEnd() throws Exception {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,15),new Range(16,25),new Range(26,27),new Range(34)});
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 15), new Range(16, 25), new Range(26, 27),
new Range(34) });
// test another type of record
line = "H2 123456 12345 12-123456";
FieldSet tokens = tokenizer.tokenize(line);
@@ -156,13 +196,13 @@ public class FixedLengthTokenizerTests {
assertEquals(line.substring(0, 5).trim(), tokens.readString(0));
assertEquals(line.substring(5, 15).trim(), tokens.readString(1));
assertEquals(line.substring(15, 25).trim(), tokens.readString(2));
assertEquals(line.substring(25, 27).trim(), tokens.readString(3));
assertEquals(line.substring(25, 27).trim(), tokens.readString(3));
}
@Test
public void testTokenizerInvalidSetup() {
tokenizer.setNames(new String[] {"a", "b"});
tokenizer.setColumns(new Range[] {new Range(1,5)});
tokenizer.setNames(new String[] { "a", "b" });
tokenizer.setColumns(new Range[] { new Range(1, 5) });
try {
tokenizer.tokenize("12345");