Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

BATCH-700:LineTokenizer implementations are now consistent in their handling of incorrect data.

Browse Source
This commit is contained in:
lucasward
2008-07-04 03:35:28 +00:00
parent e748dad82e
commit 3757b239d0
7 changed files with 254 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/AbstractLineTokenizer.java
View File

@@ -24,9 +24,12 @@ import org.springframework.batch.item.file.mapping.FieldSet;
/**
* Abstract class handling common concerns of various {@link LineTokenizer} implementations
* such as dealing with names and actual construction of {@link FieldSet}
*
* @author Dave Syer
* @author Robert Kasanicky
*
* @author Lucas Ward
*/
public abstract class AbstractLineTokenizer implements LineTokenizer {
@@ -63,19 +66,24 @@ public abstract class AbstractLineTokenizer implements LineTokenizer {
*/
public FieldSet tokenize(String line) {
if (line == null || line.length()==0) {
return new DefaultFieldSet(new String[0]);
// if (line == null || line.length()==0) {
// return new DefaultFieldSet(new String[0]);
// }
if(line == null){
line = "";
}
List tokens = new ArrayList(doTokenize(line));
for (int i=tokens.size(); i<names.length; i++) {
tokens.add(null);
}
String[] values = (String[]) tokens.toArray(new String[tokens.size()]);
if (names.length==0) {
return new DefaultFieldSet(values);
}
else if(values.length != names.length){
throw new IncorrectTokenCountException(names.length, values.length);
}
return new DefaultFieldSet(values, names);
}

26
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java
View File

@@ -26,10 +26,12 @@ import java.util.List;
* @author tomas.slanina
* @author peter.zozom
* @author Dave Syer
* @author Lucas Ward
*/
public class FixedLengthTokenizer extends AbstractLineTokenizer {
private Range[] ranges;
private int maxRange;
/**
* Set the column ranges. Used in conjunction with the
@@ -41,6 +43,23 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer {
*/
public void setColumns(Range[] ranges) {
this.ranges = ranges;
calculateMaxRange(ranges);
}
private void calculateMaxRange(Range[] ranges){
if(ranges == null || ranges.length == 0){
maxRange = 0;
return;
}
maxRange = ranges[0].getMax();
for(int i = 0; i < ranges.length; i++){
if(ranges[i].getMax() > maxRange){
maxRange = ranges[i].getMax();
}
}
}
/**
@@ -51,6 +70,8 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer {
* the line to be tokenised (can be <code>null</code>)
*
* @return the resulting tokens (empty if the line is null)
* @throws IncorrectLineLengthException if line length is greater than
* or less than the max range set.
*/
protected List doTokenize(String line) {
List tokens = new ArrayList(ranges.length);
@@ -58,6 +79,11 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer {
String token;
lineLength = line.length();
if(lineLength > maxRange || lineLength < maxRange){
//line is longer than max range, throw exception
throw new IncorrectLineLengthException(maxRange, lineLength);
}
for (int i = 0; i < ranges.length; i++) {

47
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FlatFileFormatException.java Normal file
View File

@@ -0,0 +1,47 @@
/*
* Copyright 2006-2008 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.item.file.transform;
/**
* Exception indicating that some type of error has occured while
* attempting to parse a line of input into tokens.
*
* @author Lucas Ward
*
*/
public class FlatFileFormatException extends RuntimeException {
/**
* Create a new {@link FlatFileFormatException} based on a message.
*
* @param message the message for this exception
*/
public FlatFileFormatException(String message) {
super(message);
}
/**
* Create a new {@link FlatFileFormatException} based on a message and another exception.
*
* @param message the message for this exception
* @param cause the other exception
*/
public FlatFileFormatException(String message, Throwable cause) {
super(message, cause);
}
}

49
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/IncorrectLineLengthException.java Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright 2006-2008 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.item.file.transform;
/**
* Exception indicating that the line size expected is different from what
* is expected.
*
* @author Lucas Ward
* @since 1.1
*/
public class IncorrectLineLengthException extends FlatFileFormatException {
private int actualLength;
private int expectedLength;
public IncorrectLineLengthException(String message, int expectedLength, int actualLength) {
super(message);
this.expectedLength = expectedLength;
this.actualLength = actualLength;
}
public IncorrectLineLengthException(int expectedLength, int actualLength) {
super("Incorrect line length in record: expected " + expectedLength + " actual " + actualLength);
this.actualLength = actualLength;
this.expectedLength = expectedLength;
}
public int getActualLength() {
return actualLength;
}
public int getExpectedLength() {
return expectedLength;
}
}

49
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/IncorrectTokenCountException.java Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright 2006-2008 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.item.file.transform;
/**
* Exception indicating that an incorrect number of tokens have been found
* while parsing a file.
*
* @author Lucas Ward
* @since 1.1
*/
public class IncorrectTokenCountException extends FlatFileFormatException {
private int actualCount;
private int expectedCount;
public IncorrectTokenCountException(String message, int expectedCount, int actualCount) {
super(message);
this.expectedCount = expectedCount;
this.actualCount = actualCount;
}
public IncorrectTokenCountException(int expectedCount, int actualCount) {
super("Incorrect number of tokens found in record: expected " + expectedCount + " actual " + actualCount);
this.actualCount = actualCount;
this.expectedCount = expectedCount;
}
public int getActualCount() {
return actualCount;
}
public int getExpectedCount() {
return expectedCount;
}
}

33
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java
View File

@@ -68,19 +68,24 @@ public class DelimitedLineTokenizerTests extends TestCase {
tokenizer.setNames(new String[] {"A", "B"});
try {
tokenizer.tokenize("a,b,c");
fail("Expected IllegalArgumentException");
fail("Expected IncorrectTokenCountException");
}
catch (IllegalArgumentException e) {
// expected
catch (IncorrectTokenCountException e) {
assertEquals(2, e.getExpectedCount());
assertEquals(3, e.getActualCount());
}
}
public void testTooManyNames() {
tokenizer.setNames(new String[] {"A", "B", "C", "D"});
FieldSet line = tokenizer.tokenize("a,b,c");
assertEquals(4, line.getFieldCount());
assertEquals("c", line.readString("C"));
assertEquals(null, line.readString("D"));
try{
tokenizer.tokenize("a,b,c");
}
catch(IncorrectTokenCountException e){
assertEquals(4, e.getExpectedCount());
assertEquals(3, e.getActualCount());
}
}
public void testDelimitedLineTokenizerChar() {
@@ -146,6 +151,18 @@ public class DelimitedLineTokenizerTests extends TestCase {
FieldSet line = tokenizer.tokenize("");
assertEquals(0, line.getFieldCount());
}
public void testEmptyLineWithNames(){
tokenizer.setNames(new String[]{"A", "B"});
try{
tokenizer.tokenize("");
}
catch(IncorrectTokenCountException ex){
assertEquals(2, ex.getExpectedCount());
assertEquals(0, ex.getActualCount());
}
}
public void testWhitespaceLine() throws Exception {
FieldSet line = tokenizer.tokenize(" ");

83
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java
View File

@@ -32,72 +32,79 @@ public class FixedLengthTokenizerTests extends TestCase {
*/
public void testTokenizeEmptyString() {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)});
FieldSet tokens = tokenizer.tokenize("");
assertEquals(0, tokens.getFieldCount());
try{
tokenizer.tokenize("");
}
catch(IncorrectLineLengthException ex){
assertEquals(15, ex.getExpectedLength());
assertEquals(0, ex.getActualLength());
}
}
public void testEmptyStringWithNoRanges(){
tokenizer.setColumns(new Range[]{});
tokenizer.tokenize("");
}
public void testTokenizeSmallerStringThanRanges() {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)});
FieldSet tokens = tokenizer.tokenize("12345");
assertEquals(3, tokens.getFieldCount());
assertEquals("12345", tokens.readString(0));
assertEquals("", tokens.readString(1));
assertEquals("", tokens.readString(2));
try{
tokenizer.tokenize("12345");
}
catch(IncorrectLineLengthException ex){
assertEquals(15, ex.getExpectedLength());
assertEquals(5, ex.getActualLength());
}
}
public void testTokenizeNullString() {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)});
FieldSet tokens = tokenizer.tokenize(null);
assertEquals(0, tokens.getFieldCount());
try{
tokenizer.tokenize(null);
}
catch(IncorrectLineLengthException ex){}
}
public void testTokenizeRegularUse() {
tokenizer.setColumns(new Range[] {new Range(1,2),new Range(3,7),new Range(8,12)});
// test shorter line as defined by record descriptor
line = "H1";
line = "H11234512345";
FieldSet tokens = tokenizer.tokenize(line);
assertEquals(3, tokens.getFieldCount());
assertEquals("H1", tokens.readString(0));
assertEquals("", tokens.readString(1));
assertEquals("", tokens.readString(2));
assertEquals("12345", tokens.readString(1));
assertEquals("12345", tokens.readString(2));
}
public void testNormalLength() throws Exception {
tokenizer.setColumns(new Range[] {new Range(1,10),new Range(11,25),new Range(26,30)});
// test shorter line as defined by record descriptor
line = "H1";
FieldSet tokens = tokenizer.tokenize(line);
// test normal length
line = "H1 12345678 12345";
tokens = tokenizer.tokenize(line);
FieldSet tokens = tokenizer.tokenize(line);
assertEquals(3, tokens.getFieldCount());
assertEquals(line.substring(0, 10).trim(), tokens.readString(0));
assertEquals(line.substring(10, 25).trim(), tokens.readString(1));
assertEquals(line.substring(25).trim(), tokens.readString(2));
}
public void testLongerLinesRestIgnored() throws Exception {
public void testLongerLines() throws Exception {
tokenizer.setColumns(new Range[] {new Range(1,10),new Range(11,25),new Range(26,30)});
// test shorter line as defined by record descriptor
line = "H1";
FieldSet tokens = tokenizer.tokenize(line);
// test longer lines => rest will be ignored
line = "H1 12345678 1234567890";
tokens = tokenizer.tokenize(line);
assertEquals(3, tokens.getFieldCount());
assertEquals(line.substring(0, 10).trim(), tokens.readString(0));
assertEquals(line.substring(10, 25).trim(), tokens.readString(1));
assertEquals(line.substring(25, 30).trim(), tokens.readString(2));
try{
tokenizer.tokenize(line);
}
catch(IncorrectLineLengthException ex){
assertEquals(30, ex.getExpectedLength());
assertEquals(35, ex.getActualLength());
}
}
public void testNonAdjacentRangesUnsorted() throws Exception {
tokenizer.setColumns(new Range[] {new Range(14,28), new Range(34,38), new Range(1,10)});
// test shorter line as defined by record descriptor
line = "H1";
FieldSet tokens = tokenizer.tokenize(line);
// test normal length
line = "H1 +++12345678 +++++12345+++";
tokens = tokenizer.tokenize(line);
line = "H1 +++12345678 +++++12345";
FieldSet tokens = tokenizer.tokenize(line);
assertEquals(3, tokens.getFieldCount());
assertEquals(line.substring(0, 10).trim(), tokens.readString(2));
assertEquals(line.substring(13, 28).trim(), tokens.readString(0));
@@ -106,12 +113,9 @@ public class FixedLengthTokenizerTests extends TestCase {
public void testAnotherTypeOfRecord() throws Exception {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,15),new Range(16,25),new Range(26,27)});
// test shorter line as defined by record descriptor
line = "H1";
FieldSet tokens = tokenizer.tokenize(line);
// test another type of record
line = "H2 123456 12345 12";
tokens = tokenizer.tokenize(line);
FieldSet tokens = tokenizer.tokenize(line);
assertEquals(4, tokens.getFieldCount());
assertEquals(line.substring(0, 5).trim(), tokens.readString(0));
assertEquals(line.substring(5, 15).trim(), tokens.readString(1));
@@ -121,14 +125,15 @@ public class FixedLengthTokenizerTests extends TestCase {
public void testTokenizerInvalidSetup() {
tokenizer.setNames(new String[] {"a", "b"});
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,15),new Range(16,25),new Range(26,27)});
tokenizer.setColumns(new Range[] {new Range(1,5)});
try {
tokenizer.tokenize("Test tokenize");
tokenizer.tokenize("12345");
fail("Exception was expected: too few names provided");
}
catch (Exception e) {
assertTrue(true);
catch (IncorrectTokenCountException e) {
assertEquals(2, e.getExpectedCount());
assertEquals(1, e.getActualCount());
}
}