diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java index 29fc83024..242fa1d20 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java @@ -142,14 +142,16 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { int lastCut = 0; int length = chars.length; int fieldCount = 0; + int endIndexLastDelimiter = -1; for (int i = 0; i < length; i++) { char currentChar = chars[i]; boolean isEnd = (i == (length - 1)); - boolean isDelimiter = isDelimiter(chars, i, delimiter); + boolean isDelimiter = isDelimiter(chars, i, delimiter, endIndexLastDelimiter); if ((isDelimiter && !inQuoted) || isEnd) { + endIndexLastDelimiter = i; int endPosition = (isEnd ? (length - lastCut) : (i - lastCut)); if (isEnd && isDelimiter) { @@ -229,13 +231,15 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { * character * @see DelimitedLineTokenizer#DelimitedLineTokenizer(char) */ - private boolean isDelimiter(char[] chars, int i, String token) { + private boolean isDelimiter(char[] chars, int i, String token, int endIndexLastDelimiter) { boolean result = false; - if(i >= token.length() - 1) { - String end = new String(chars, (i-token.length()) + 1, token.length()); - if(token.equals(end)) { - result = true; + if(i-endIndexLastDelimiter >= delimiter.length()) { + if(i >= token.length() - 1) { + String end = new String(chars, (i-token.length()) + 1, token.length()); + if(token.equals(end)) { + result = true; + } } } diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java index b8e725717..eb687f0af 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java @@ -158,6 +158,25 @@ public class DelimitedLineTokenizerTests { assertEquals("", line.readString(2)); } + @Test + public void testDelimitedLineTokenizerStringsOverlap() { + AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" | "); + FieldSet line = tokenizer.tokenize("a | | | b"); + assertEquals(3, line.getFieldCount()); + assertEquals("a", line.readString(0)); + assertEquals("|", line.readString(1)); + assertEquals("b", line.readString(2)); + } + + @Test + public void testDelimitedLineTokenizerStringsOverlapWithoutSeparation() { + AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" | "); + FieldSet line = tokenizer.tokenize("a | | b"); + assertEquals(2, line.getFieldCount()); + assertEquals("a", line.readString(0)); + assertEquals("| b", line.readString(1)); + } + @Test public void testDelimitedLineTokenizerNewlineToken() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer("\n");