From 3757b239d007483bb4f480012377b4095ce9e0ec Mon Sep 17 00:00:00 2001 From: lucasward Date: Fri, 4 Jul 2008 03:35:28 +0000 Subject: [PATCH] BATCH-700:LineTokenizer implementations are now consistent in their handling of incorrect data. --- .../file/transform/AbstractLineTokenizer.java | 20 +++-- .../file/transform/FixedLengthTokenizer.java | 26 ++++++ .../transform/FlatFileFormatException.java | 47 +++++++++++ .../IncorrectLineLengthException.java | 49 +++++++++++ .../IncorrectTokenCountException.java | 49 +++++++++++ .../DelimitedLineTokenizerTests.java | 33 ++++++-- .../transform/FixedLengthTokenizerTests.java | 83 ++++++++++--------- 7 files changed, 254 insertions(+), 53 deletions(-) create mode 100644 spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FlatFileFormatException.java create mode 100644 spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/IncorrectLineLengthException.java create mode 100644 spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/IncorrectTokenCountException.java diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/AbstractLineTokenizer.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/AbstractLineTokenizer.java index 586ffbf93..66944fad5 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/AbstractLineTokenizer.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/AbstractLineTokenizer.java @@ -24,9 +24,12 @@ import org.springframework.batch.item.file.mapping.FieldSet; /** + * Abstract class handling common concerns of various {@link LineTokenizer} implementations + * such as dealing with names and actual construction of {@link FieldSet} + * * @author Dave Syer * @author Robert Kasanicky - * + * @author Lucas Ward */ public abstract class AbstractLineTokenizer implements LineTokenizer { @@ -63,19 +66,24 @@ public abstract class AbstractLineTokenizer implements LineTokenizer { */ public FieldSet tokenize(String line) { - if (line == null || line.length()==0) { - return new DefaultFieldSet(new String[0]); +// if (line == null || line.length()==0) { +// return new DefaultFieldSet(new String[0]); +// } + + if(line == null){ + line = ""; } List tokens = new ArrayList(doTokenize(line)); - for (int i=tokens.size(); i maxRange){ + maxRange = ranges[i].getMax(); + } + } } /** @@ -51,6 +70,8 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer { * the line to be tokenised (can be null) * * @return the resulting tokens (empty if the line is null) + * @throws IncorrectLineLengthException if line length is greater than + * or less than the max range set. */ protected List doTokenize(String line) { List tokens = new ArrayList(ranges.length); @@ -58,6 +79,11 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer { String token; lineLength = line.length(); + + if(lineLength > maxRange || lineLength < maxRange){ + //line is longer than max range, throw exception + throw new IncorrectLineLengthException(maxRange, lineLength); + } for (int i = 0; i < ranges.length; i++) { diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FlatFileFormatException.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FlatFileFormatException.java new file mode 100644 index 000000000..a8e3e3661 --- /dev/null +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FlatFileFormatException.java @@ -0,0 +1,47 @@ +/* + * Copyright 2006-2008 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.item.file.transform; + + + +/** + * Exception indicating that some type of error has occured while + * attempting to parse a line of input into tokens. + * + * @author Lucas Ward + * + */ +public class FlatFileFormatException extends RuntimeException { + + /** + * Create a new {@link FlatFileFormatException} based on a message. + * + * @param message the message for this exception + */ + public FlatFileFormatException(String message) { + super(message); + } + + /** + * Create a new {@link FlatFileFormatException} based on a message and another exception. + * + * @param message the message for this exception + * @param cause the other exception + */ + public FlatFileFormatException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/IncorrectLineLengthException.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/IncorrectLineLengthException.java new file mode 100644 index 000000000..baff68f3e --- /dev/null +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/IncorrectLineLengthException.java @@ -0,0 +1,49 @@ +/* + * Copyright 2006-2008 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.item.file.transform; + +/** + * Exception indicating that the line size expected is different from what + * is expected. + * + * @author Lucas Ward + * @since 1.1 + */ +public class IncorrectLineLengthException extends FlatFileFormatException { + + private int actualLength; + private int expectedLength; + + public IncorrectLineLengthException(String message, int expectedLength, int actualLength) { + super(message); + this.expectedLength = expectedLength; + this.actualLength = actualLength; + } + + public IncorrectLineLengthException(int expectedLength, int actualLength) { + super("Incorrect line length in record: expected " + expectedLength + " actual " + actualLength); + this.actualLength = actualLength; + this.expectedLength = expectedLength; + } + + public int getActualLength() { + return actualLength; + } + + public int getExpectedLength() { + return expectedLength; + } +} diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/IncorrectTokenCountException.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/IncorrectTokenCountException.java new file mode 100644 index 000000000..f7cf3abd6 --- /dev/null +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/IncorrectTokenCountException.java @@ -0,0 +1,49 @@ +/* + * Copyright 2006-2008 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.item.file.transform; + +/** + * Exception indicating that an incorrect number of tokens have been found + * while parsing a file. + * + * @author Lucas Ward + * @since 1.1 + */ +public class IncorrectTokenCountException extends FlatFileFormatException { + + private int actualCount; + private int expectedCount; + + public IncorrectTokenCountException(String message, int expectedCount, int actualCount) { + super(message); + this.expectedCount = expectedCount; + this.actualCount = actualCount; + } + + public IncorrectTokenCountException(int expectedCount, int actualCount) { + super("Incorrect number of tokens found in record: expected " + expectedCount + " actual " + actualCount); + this.actualCount = actualCount; + this.expectedCount = expectedCount; + } + + public int getActualCount() { + return actualCount; + } + + public int getExpectedCount() { + return expectedCount; + } +} diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java index c5ded246f..925d97488 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java @@ -68,19 +68,24 @@ public class DelimitedLineTokenizerTests extends TestCase { tokenizer.setNames(new String[] {"A", "B"}); try { tokenizer.tokenize("a,b,c"); - fail("Expected IllegalArgumentException"); + fail("Expected IncorrectTokenCountException"); } - catch (IllegalArgumentException e) { - // expected + catch (IncorrectTokenCountException e) { + assertEquals(2, e.getExpectedCount()); + assertEquals(3, e.getActualCount()); } } - + public void testTooManyNames() { tokenizer.setNames(new String[] {"A", "B", "C", "D"}); - FieldSet line = tokenizer.tokenize("a,b,c"); - assertEquals(4, line.getFieldCount()); - assertEquals("c", line.readString("C")); - assertEquals(null, line.readString("D")); + try{ + tokenizer.tokenize("a,b,c"); + } + catch(IncorrectTokenCountException e){ + assertEquals(4, e.getExpectedCount()); + assertEquals(3, e.getActualCount()); + } + } public void testDelimitedLineTokenizerChar() { @@ -146,6 +151,18 @@ public class DelimitedLineTokenizerTests extends TestCase { FieldSet line = tokenizer.tokenize(""); assertEquals(0, line.getFieldCount()); } + + public void testEmptyLineWithNames(){ + + tokenizer.setNames(new String[]{"A", "B"}); + try{ + tokenizer.tokenize(""); + } + catch(IncorrectTokenCountException ex){ + assertEquals(2, ex.getExpectedCount()); + assertEquals(0, ex.getActualCount()); + } + } public void testWhitespaceLine() throws Exception { FieldSet line = tokenizer.tokenize(" "); diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java index 4d4a29f32..71b784980 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java @@ -32,72 +32,79 @@ public class FixedLengthTokenizerTests extends TestCase { */ public void testTokenizeEmptyString() { tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)}); - FieldSet tokens = tokenizer.tokenize(""); - assertEquals(0, tokens.getFieldCount()); + try{ + tokenizer.tokenize(""); + } + catch(IncorrectLineLengthException ex){ + assertEquals(15, ex.getExpectedLength()); + assertEquals(0, ex.getActualLength()); + } + } + + public void testEmptyStringWithNoRanges(){ + tokenizer.setColumns(new Range[]{}); + tokenizer.tokenize(""); } public void testTokenizeSmallerStringThanRanges() { tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)}); - FieldSet tokens = tokenizer.tokenize("12345"); - assertEquals(3, tokens.getFieldCount()); - assertEquals("12345", tokens.readString(0)); - assertEquals("", tokens.readString(1)); - assertEquals("", tokens.readString(2)); + try{ + tokenizer.tokenize("12345"); + } + catch(IncorrectLineLengthException ex){ + assertEquals(15, ex.getExpectedLength()); + assertEquals(5, ex.getActualLength()); + } + } public void testTokenizeNullString() { tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)}); - FieldSet tokens = tokenizer.tokenize(null); - assertEquals(0, tokens.getFieldCount()); + try{ + tokenizer.tokenize(null); + } + catch(IncorrectLineLengthException ex){} } public void testTokenizeRegularUse() { tokenizer.setColumns(new Range[] {new Range(1,2),new Range(3,7),new Range(8,12)}); // test shorter line as defined by record descriptor - line = "H1"; + line = "H11234512345"; FieldSet tokens = tokenizer.tokenize(line); assertEquals(3, tokens.getFieldCount()); assertEquals("H1", tokens.readString(0)); - assertEquals("", tokens.readString(1)); - assertEquals("", tokens.readString(2)); + assertEquals("12345", tokens.readString(1)); + assertEquals("12345", tokens.readString(2)); } public void testNormalLength() throws Exception { tokenizer.setColumns(new Range[] {new Range(1,10),new Range(11,25),new Range(26,30)}); // test shorter line as defined by record descriptor - line = "H1"; - FieldSet tokens = tokenizer.tokenize(line); - // test normal length line = "H1 12345678 12345"; - tokens = tokenizer.tokenize(line); + FieldSet tokens = tokenizer.tokenize(line); assertEquals(3, tokens.getFieldCount()); assertEquals(line.substring(0, 10).trim(), tokens.readString(0)); assertEquals(line.substring(10, 25).trim(), tokens.readString(1)); assertEquals(line.substring(25).trim(), tokens.readString(2)); } - public void testLongerLinesRestIgnored() throws Exception { + public void testLongerLines() throws Exception { tokenizer.setColumns(new Range[] {new Range(1,10),new Range(11,25),new Range(26,30)}); - // test shorter line as defined by record descriptor - line = "H1"; - FieldSet tokens = tokenizer.tokenize(line); - // test longer lines => rest will be ignored line = "H1 12345678 1234567890"; - tokens = tokenizer.tokenize(line); - assertEquals(3, tokens.getFieldCount()); - assertEquals(line.substring(0, 10).trim(), tokens.readString(0)); - assertEquals(line.substring(10, 25).trim(), tokens.readString(1)); - assertEquals(line.substring(25, 30).trim(), tokens.readString(2)); + try{ + tokenizer.tokenize(line); + } + catch(IncorrectLineLengthException ex){ + assertEquals(30, ex.getExpectedLength()); + assertEquals(35, ex.getActualLength()); + } } public void testNonAdjacentRangesUnsorted() throws Exception { tokenizer.setColumns(new Range[] {new Range(14,28), new Range(34,38), new Range(1,10)}); - // test shorter line as defined by record descriptor - line = "H1"; - FieldSet tokens = tokenizer.tokenize(line); // test normal length - line = "H1 +++12345678 +++++12345+++"; - tokens = tokenizer.tokenize(line); + line = "H1 +++12345678 +++++12345"; + FieldSet tokens = tokenizer.tokenize(line); assertEquals(3, tokens.getFieldCount()); assertEquals(line.substring(0, 10).trim(), tokens.readString(2)); assertEquals(line.substring(13, 28).trim(), tokens.readString(0)); @@ -106,12 +113,9 @@ public class FixedLengthTokenizerTests extends TestCase { public void testAnotherTypeOfRecord() throws Exception { tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,15),new Range(16,25),new Range(26,27)}); - // test shorter line as defined by record descriptor - line = "H1"; - FieldSet tokens = tokenizer.tokenize(line); // test another type of record line = "H2 123456 12345 12"; - tokens = tokenizer.tokenize(line); + FieldSet tokens = tokenizer.tokenize(line); assertEquals(4, tokens.getFieldCount()); assertEquals(line.substring(0, 5).trim(), tokens.readString(0)); assertEquals(line.substring(5, 15).trim(), tokens.readString(1)); @@ -121,14 +125,15 @@ public class FixedLengthTokenizerTests extends TestCase { public void testTokenizerInvalidSetup() { tokenizer.setNames(new String[] {"a", "b"}); - tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,15),new Range(16,25),new Range(26,27)}); + tokenizer.setColumns(new Range[] {new Range(1,5)}); try { - tokenizer.tokenize("Test tokenize"); + tokenizer.tokenize("12345"); fail("Exception was expected: too few names provided"); } - catch (Exception e) { - assertTrue(true); + catch (IncorrectTokenCountException e) { + assertEquals(2, e.getExpectedCount()); + assertEquals(1, e.getActualCount()); } }