From 15fbb6f9acb3f6215349b77f81c238b78d15b882 Mon Sep 17 00:00:00 2001 From: dsyer Date: Wed, 4 Nov 2009 09:36:17 +0000 Subject: [PATCH] RESOLVED - issue BATCH-1429: Allow DelimitedLineTokenizer to handle malformed file/lines gracefully. --- .../file/transform/AbstractLineTokenizer.java | 65 ++++++++++++++++++- .../file/transform/FixedLengthTokenizer.java | 20 +----- .../DelimitedLineTokenizerTests.java | 23 +++++++ 3 files changed, 89 insertions(+), 19 deletions(-) diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/AbstractLineTokenizer.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/AbstractLineTokenizer.java index 08f75f55c..dae65e9f1 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/AbstractLineTokenizer.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/AbstractLineTokenizer.java @@ -32,8 +32,35 @@ public abstract class AbstractLineTokenizer implements LineTokenizer { protected String[] names = new String[0]; + private boolean strict = true; + + private String emptyToken = ""; + private FieldSetFactory fieldSetFactory = new DefaultFieldSetFactory(); + /** + * Public setter for the strict flag. If true (the default) then number of + * tokens in line must match the number of tokens defined + * (by {@link Range}, columns, etc.) in {@link LineTokenizer}. + * If false then lines with less tokens will be tolerated and padded with + * empty columns, and lines with more tokens will + * simply be truncated. + * + * @param strict the strict flag to set + */ + public void setStrict(boolean strict) { + this.strict = strict; + } + + /** + * Provides access to the strict flag for subclasses if needed. + * + * @return the strict flag value + */ + protected boolean isStrict() { + return strict; + } + /** * Factory for {@link FieldSet} instances. Can be injected by clients to * customize the default number and date formats. @@ -80,7 +107,12 @@ public abstract class AbstractLineTokenizer implements LineTokenizer { } List tokens = new ArrayList(doTokenize(line)); - + + // if names are set and strict flag is false + if ( ( names.length != 0 ) && ( ! strict ) ) { + adjustTokenCountIfNecessary( tokens ); + } + String[] values = (String[]) tokens.toArray(new String[tokens.size()]); if (names.length == 0) { @@ -93,5 +125,36 @@ public abstract class AbstractLineTokenizer implements LineTokenizer { } protected abstract List doTokenize(String line); + + /** + * Adds empty tokens or truncates existing token list to match expected + * (configured) number of tokens in {@link LineTokenizer}. + * + * @param tokens - list of tokens + */ + private void adjustTokenCountIfNecessary( List tokens ) { + + int nameLength = names.length; + int tokensSize = tokens.size(); + + // if the number of tokens is not what expected + if ( nameLength != tokensSize ) { + + if ( nameLength > tokensSize ) { + // add empty tokens until the token list size matches + // the expected number of tokens + for ( int i = 0; i < ( nameLength - tokensSize ); i++ ) { + tokens.add( emptyToken ); + } + + } else { + // truncate token list to match the number of expected tokens + for ( int i = tokensSize - 1; i >= nameLength; i-- ) { + tokens.remove(i); + } + } + + } + } } diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java index 50b7ea787..1a85073f4 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/FixedLengthTokenizer.java @@ -37,22 +37,6 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer { boolean open = false; - private boolean strict = true; - - /** - * Public setter for the strict flag. If true (the default) then lines must - * be precisely the length specified by the columns. If false then shorter - * lines will be tolerated and padded with empty columns, and longer strings - * will simply be truncated. - * - * @see #setColumns(Range[]) - * - * @param strict the strict to set - */ - public void setStrict(boolean strict) { - this.strict = strict; - } - /** * Set the column ranges. Used in conjunction with the * {@link RangeArrayPropertyEditor} this property can be set in the form of @@ -118,11 +102,11 @@ public class FixedLengthTokenizer extends AbstractLineTokenizer { lineLength = line.length(); - if (lineLength < maxRange && strict) { + if (lineLength < maxRange && isStrict()) { throw new IncorrectLineLengthException("Line is shorter than max range " + maxRange, maxRange, lineLength); } - if (!open && lineLength > maxRange && strict) { + if (!open && lineLength > maxRange && isStrict()) { throw new IncorrectLineLengthException("Line is longer than max range " + maxRange, maxRange, lineLength); } diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java index 5de4864ad..6b60d29d7 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java @@ -75,6 +75,16 @@ public class DelimitedLineTokenizerTests extends TestCase { } } + public void testTooFewNamesNotStrict() { + tokenizer.setNames(new String[] {"A", "B"}); + tokenizer.setStrict(false); + + FieldSet tokens = tokenizer.tokenize("a,b,c"); + + assertTrue(TOKEN_MATCHES, tokens.readString(0).equals("a")); + assertTrue(TOKEN_MATCHES, tokens.readString(1).equals("b")); + } + public void testTooManyNames() { tokenizer.setNames(new String[] {"A", "B", "C", "D"}); try{ @@ -86,6 +96,19 @@ public class DelimitedLineTokenizerTests extends TestCase { } } + + public void testTooManyNamesNotStrict() { + tokenizer.setNames(new String[] {"A", "B", "C", "D","E"}); + tokenizer.setStrict( false ); + + FieldSet tokens = tokenizer.tokenize("a,b,c"); + + assertTrue(TOKEN_MATCHES, tokens.readString(0).equals("a")); + assertTrue(TOKEN_MATCHES, tokens.readString(1).equals("b")); + assertTrue(TOKEN_MATCHES, tokens.readString(2).equals("c")); + assertTrue(TOKEN_MATCHES, tokens.readString(3).equals("")); + assertTrue(TOKEN_MATCHES, tokens.readString(4).equals("")); + } public void testDelimitedLineTokenizerChar() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(' ');