diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java
index d94dc08f5..fd53c4eb2 100644
--- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java
+++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2006-2007 the original author or authors.
+ * Copyright 2006-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,21 +28,22 @@ import org.springframework.util.StringUtils;
* A {@link LineTokenizer} implementation that splits the input String on a
* configurable delimiter. This implementation also supports the use of an
* escape character to escape delimiters and line endings.
- *
+ *
* @author Rob Harrop
* @author Dave Syer
- *
+ * @author Michael Minella
*/
+@Deprecated
public class DelimitedLineTokenizer extends AbstractLineTokenizer {
/**
* Convenient constant for the common case of a tab delimiter.
*/
- public static final char DELIMITER_TAB = '\t';
+ public static final String DELIMITER_TAB = "\t";
/**
* Convenient constant for the common case of a comma delimiter.
*/
- public static final char DELIMITER_COMMA = ',';
+ public static final String DELIMITER_COMMA = ",";
/**
* Convenient constant for the common case of a " character used to escape
@@ -51,7 +52,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
public static final char DEFAULT_QUOTE_CHARACTER = '"';
// the delimiter character used when reading input.
- private char delimiter;
+ private String delimiter;
private char quoteCharacter = DEFAULT_QUOTE_CHARACTER;
@@ -62,7 +63,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
/**
* Create a new instance of the {@link DelimitedLineTokenizer} class for the
* common case where the delimiter is a {@link #DELIMITER_COMMA comma}.
- *
+ *
* @see #DelimitedLineTokenizer(char)
* @see #DELIMITER_COMMA
*/
@@ -72,11 +73,11 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
/**
* Create a new instance of the {@link DelimitedLineTokenizer} class.
- *
+ *
* @param delimiter the desired delimiter
*/
- public DelimitedLineTokenizer(char delimiter) {
- Assert.state(delimiter != DEFAULT_QUOTE_CHARACTER, "[" + DEFAULT_QUOTE_CHARACTER
+ public DelimitedLineTokenizer(String delimiter) {
+ Assert.state(!delimiter.equals(String.valueOf(DEFAULT_QUOTE_CHARACTER)), "[" + DEFAULT_QUOTE_CHARACTER
+ "] is not allowed as delimiter for tokenizers.");
this.delimiter = delimiter;
@@ -85,10 +86,10 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
/**
* Setter for the delimiter character.
- *
+ *
* @param delimiter
*/
- public void setDelimiter(char delimiter) {
+ public void setDelimiter(String delimiter) {
this.delimiter = delimiter;
}
@@ -97,7 +98,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
* default all fields are included, but this property can be set to pick out
* only a few fields from a larger set. Note that if field names are
* provided, their number must match the number of included fields.
- *
+ *
* @param includedFields the included fields to set
*/
public void setIncludedFields(int[] includedFields) {
@@ -112,9 +113,9 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
* extend a field across line endings or to enclose a String which contains
* the delimiter. Inside a quoted token the quote character can be used to
* escape itself, thus "a""b""c" is tokenized to a"b"c.
- *
+ *
* @param quoteCharacter the quoteCharacter to set
- *
+ *
* @see #DEFAULT_QUOTE_CHARACTER
*/
public final void setQuoteCharacter(char quoteCharacter) {
@@ -125,11 +126,12 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
/**
* Yields the tokens resulting from the splitting of the supplied
* line.
- *
+ *
* @param line the line to be tokenized
- *
+ *
* @return the resulting tokens
*/
+ @Override
protected List doTokenize(String line) {
List tokens = new ArrayList();
@@ -147,12 +149,17 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
char currentChar = chars[i];
boolean isEnd = (i == (length - 1));
- if ((isDelimiterCharacter(currentChar) && !inQuoted) || isEnd) {
+ boolean isDelimiter = isDelimiter(chars, i, delimiter);
+
+ if ((isDelimiter && !inQuoted) || isEnd) {
int endPosition = (isEnd ? (length - lastCut) : (i - lastCut));
- if (isEnd && isDelimiterCharacter(currentChar)) {
+ if (isEnd && isDelimiter) {
endPosition--;
}
+ else if (!isEnd){
+ endPosition = (endPosition - delimiter.length()) + 1;
+ }
if (includedFields == null || includedFields.contains(fieldCount)) {
String value = maybeStripQuotes(new String(chars, lastCut, endPosition));
@@ -161,7 +168,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
fieldCount++;
- if (isEnd && (isDelimiterCharacter(currentChar))) {
+ if (isEnd && (isDelimiter)) {
if (includedFields == null || includedFields.contains(fieldCount)) {
tokens.add("");
}
@@ -183,7 +190,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
* If the string is quoted strip (possibly with whitespace outside the
* quotes (which will be stripped), replace escaped quotes inside the
* string. Quotes are escaped with double instances of the quote character.
- *
+ *
* @param string
* @return the same string but stripped and unescaped if necessary
*/
@@ -204,7 +211,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
/**
* Is this string surrounded by quote characters?
- *
+ *
* @param value
* @return true if the value starts and ends with the
* {@link #quoteCharacter}
@@ -218,19 +225,28 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
/**
* Is the supplied character the delimiter character?
- *
+ *
* @param c the character to be checked
* @return true if the supplied character is the delimiter
* character
* @see DelimitedLineTokenizer#DelimitedLineTokenizer(char)
*/
- private boolean isDelimiterCharacter(char c) {
- return c == this.delimiter;
+ private boolean isDelimiter(char[] chars, int i, String token) {
+ boolean result = false;
+
+ if(i >= token.length()) {
+ String end = new String(chars, (i-token.length()) + 1, token.length());
+ if(token.equals(end)) {
+ result = true;
+ }
+ }
+
+ return result;
}
/**
* Is the supplied character a quote character?
- *
+ *
* @param c the character to be checked
* @return true if the supplied character is an quote character
* @see #setQuoteCharacter(char)
diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java
index 9ec4d7ba9..ab1e7093a 100644
--- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java
+++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2006-2007 the original author or authors.
+ * Copyright 2006-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -49,7 +49,7 @@ public class DelimitedLineTokenizerTests {
@Test
public void testInvalidConstructorArgument() {
try {
- new DelimitedLineTokenizer(DelimitedLineTokenizer.DEFAULT_QUOTE_CHARACTER);
+ new DelimitedLineTokenizer(String.valueOf(DelimitedLineTokenizer.DEFAULT_QUOTE_CHARACTER));
fail("Quote character can't be used as delimiter for delimited line tokenizer!");
}
catch (Exception e) {
@@ -83,18 +83,18 @@ public class DelimitedLineTokenizerTests {
assertEquals(3, e.getActualCount());
}
}
-
+
@Test
public void testTooFewNamesNotStrict() {
tokenizer.setNames(new String[] {"A", "B"});
tokenizer.setStrict(false);
FieldSet tokens = tokenizer.tokenize("a,b,c");
-
+
assertTrue(TOKEN_MATCHES, tokens.readString(0).equals("a"));
assertTrue(TOKEN_MATCHES, tokens.readString(1).equals("b"));
- }
-
+ }
+
@Test
public void testTooManyNames() {
tokenizer.setNames(new String[] {"A", "B", "C", "D"});
@@ -105,30 +105,57 @@ public class DelimitedLineTokenizerTests {
assertEquals(4, e.getExpectedCount());
assertEquals(3, e.getActualCount());
}
-
+
}
-
+
@Test
public void testTooManyNamesNotStrict() {
tokenizer.setNames(new String[] {"A", "B", "C", "D","E"});
tokenizer.setStrict( false );
FieldSet tokens = tokenizer.tokenize("a,b,c");
-
+
assertTrue(TOKEN_MATCHES, tokens.readString(0).equals("a"));
assertTrue(TOKEN_MATCHES, tokens.readString(1).equals("b"));
assertTrue(TOKEN_MATCHES, tokens.readString(2).equals("c"));
assertTrue(TOKEN_MATCHES, tokens.readString(3).equals(""));
- assertTrue(TOKEN_MATCHES, tokens.readString(4).equals(""));
- }
+ assertTrue(TOKEN_MATCHES, tokens.readString(4).equals(""));
+ }
@Test
public void testDelimitedLineTokenizerChar() {
- AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(' ');
+ AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" ");
FieldSet line = tokenizer.tokenize("a b c");
assertEquals(3, line.getFieldCount());
}
+ @Test
+ public void testDelimitedLineTokenizerString() {
+ AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" b ");
+ FieldSet line = tokenizer.tokenize("a b c");
+ assertEquals(2, line.getFieldCount());
+ assertEquals("a", line.readString(0));
+ assertEquals("c", line.readString(1));
+ }
+
+ @Test
+ public void testDelimitedLineTokenizerNewlineToken() {
+ AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer("\n");
+ FieldSet line = tokenizer.tokenize("a b\n c");
+ assertEquals(2, line.getFieldCount());
+ assertEquals("a b", line.readString(0));
+ assertEquals("c", line.readString(1));
+ }
+
+ @Test
+ public void testDelimitedLineTokenizerWrappedToken() {
+ AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer("\nrap");
+ FieldSet line = tokenizer.tokenize("a b\nrap c");
+ assertEquals(2, line.getFieldCount());
+ assertEquals("a b", line.readString(0));
+ assertEquals("c", line.readString(1));
+ }
+
@Test
public void testTokenizeWithQuotes() {
FieldSet line = tokenizer.tokenize("a,b,\"c\"");
@@ -202,10 +229,10 @@ public class DelimitedLineTokenizerTests {
FieldSet line = tokenizer.tokenize("");
assertEquals(0, line.getFieldCount());
}
-
+
@Test
public void testEmptyLineWithNames(){
-
+
tokenizer.setNames(new String[]{"A", "B"});
try{
tokenizer.tokenize("");
@@ -245,7 +272,7 @@ public class DelimitedLineTokenizerTests {
assertEquals("c\nrap", line.readString(2));
}
-
+
@Test
public void testTokenizeWithQuotesEmptyValue() {
FieldSet line = tokenizer.tokenize("\"a\",\"b\",\"\",\"d\"");
@@ -296,4 +323,4 @@ public class DelimitedLineTokenizerTests {
assertEquals("c", line.readString("bar"));
}
-}
+}
\ No newline at end of file