Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

BATCH-2031: Addressed the use case of overlapping delimiters

Browse Source
This commit is contained in:
Michael Minella
2013-06-04 10:23:17 -05:00
parent 0f03ea0ffa
commit 4a32a9c953
2 changed files with 29 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java
View File

@@ -142,14 +142,16 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
int lastCut = 0;
int length = chars.length;
int fieldCount = 0;
int endIndexLastDelimiter = -1;
for (int i = 0; i < length; i++) {
char currentChar = chars[i];
boolean isEnd = (i == (length - 1));
boolean isDelimiter = isDelimiter(chars, i, delimiter);
boolean isDelimiter = isDelimiter(chars, i, delimiter, endIndexLastDelimiter);
if ((isDelimiter && !inQuoted) || isEnd) {
endIndexLastDelimiter = i;
int endPosition = (isEnd ? (length - lastCut) : (i - lastCut));
if (isEnd && isDelimiter) {
@@ -229,13 +231,15 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
* character
* @see DelimitedLineTokenizer#DelimitedLineTokenizer(char)
*/
private boolean isDelimiter(char[] chars, int i, String token) {
private boolean isDelimiter(char[] chars, int i, String token, int endIndexLastDelimiter) {
boolean result = false;
if(i >= token.length() - 1) {
String end = new String(chars, (i-token.length()) + 1, token.length());
if(token.equals(end)) {
result = true;
if(i-endIndexLastDelimiter >= delimiter.length()) {
if(i >= token.length() - 1) {
String end = new String(chars, (i-token.length()) + 1, token.length());
if(token.equals(end)) {
result = true;
}
}
}

19
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java
View File

@@ -158,6 +158,25 @@ public class DelimitedLineTokenizerTests {
assertEquals("", line.readString(2));
}
@Test
public void testDelimitedLineTokenizerStringsOverlap() {
AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" | ");
FieldSet line = tokenizer.tokenize("a | | | b");
assertEquals(3, line.getFieldCount());
assertEquals("a", line.readString(0));
assertEquals("|", line.readString(1));
assertEquals("b", line.readString(2));
}
@Test
public void testDelimitedLineTokenizerStringsOverlapWithoutSeparation() {
AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" | ");
FieldSet line = tokenizer.tokenize("a | | b");
assertEquals(2, line.getFieldCount());
assertEquals("a", line.readString(0));
assertEquals("| b", line.readString(1));
}
@Test
public void testDelimitedLineTokenizerNewlineToken() {
AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer("\n");