BATCH-1869: Updated DelimitedLineTokenizer to support String delimiters

spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2006-2007 the original author or authors.
+ * Copyright 2006-2012 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,21 +28,22 @@ import org.springframework.util.StringUtils;
  * A {@link LineTokenizer} implementation that splits the input String on a
  * configurable delimiter. This implementation also supports the use of an
  * escape character to escape delimiters and line endings.
- * 
+ *
  * @author Rob Harrop
  * @author Dave Syer
- * 
+ * @author Michael Minella
  */
+@Deprecated
 public class DelimitedLineTokenizer extends AbstractLineTokenizer {
 	/**
 	 * Convenient constant for the common case of a tab delimiter.
 	 */
-	public static final char DELIMITER_TAB = '\t';
+	public static final String DELIMITER_TAB = "\t";
 
 	/**
 	 * Convenient constant for the common case of a comma delimiter.
 	 */
-	public static final char DELIMITER_COMMA = ',';
+	public static final String DELIMITER_COMMA = ",";
 
 	/**
 	 * Convenient constant for the common case of a " character used to escape
@@ -51,7 +52,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
 	public static final char DEFAULT_QUOTE_CHARACTER = '"';
 
 	// the delimiter character used when reading input.
-	private char delimiter;
+	private String delimiter;
 
 	private char quoteCharacter = DEFAULT_QUOTE_CHARACTER;
 
@@ -62,7 +63,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
 	/**
 	 * Create a new instance of the {@link DelimitedLineTokenizer} class for the
 	 * common case where the delimiter is a {@link #DELIMITER_COMMA comma}.
-	 * 
+	 *
 	 * @see #DelimitedLineTokenizer(char)
 	 * @see #DELIMITER_COMMA
 	 */
@@ -72,11 +73,11 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
 
 	/**
 	 * Create a new instance of the {@link DelimitedLineTokenizer} class.
-	 * 
+	 *
 	 * @param delimiter the desired delimiter
 	 */
-	public DelimitedLineTokenizer(char delimiter) {
-		Assert.state(delimiter != DEFAULT_QUOTE_CHARACTER, "[" + DEFAULT_QUOTE_CHARACTER
+	public DelimitedLineTokenizer(String delimiter) {
+		Assert.state(!delimiter.equals(String.valueOf(DEFAULT_QUOTE_CHARACTER)), "[" + DEFAULT_QUOTE_CHARACTER
 				+ "] is not allowed as delimiter for tokenizers.");
 
 		this.delimiter = delimiter;
@@ -85,10 +86,10 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
 
 	/**
 	 * Setter for the delimiter character.
-	 * 
+	 *
 	 * @param delimiter
 	 */
-	public void setDelimiter(char delimiter) {
+	public void setDelimiter(String delimiter) {
 		this.delimiter = delimiter;
 	}
 
@@ -97,7 +98,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
 	 * default all fields are included, but this property can be set to pick out
 	 * only a few fields from a larger set. Note that if field names are
 	 * provided, their number must match the number of included fields.
-	 * 
+	 *
 	 * @param includedFields the included fields to set
 	 */
 	public void setIncludedFields(int[] includedFields) {
@@ -112,9 +113,9 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
 	 * extend a field across line endings or to enclose a String which contains
 	 * the delimiter. Inside a quoted token the quote character can be used to
 	 * escape itself, thus "a""b""c" is tokenized to a"b"c.
-	 * 
+	 *
 	 * @param quoteCharacter the quoteCharacter to set
-	 * 
+	 *
 	 * @see #DEFAULT_QUOTE_CHARACTER
 	 */
 	public final void setQuoteCharacter(char quoteCharacter) {
@@ -125,11 +126,12 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
 	/**
 	 * Yields the tokens resulting from the splitting of the supplied
 	 * <code>line</code>.
-	 * 
+	 *
 	 * @param line the line to be tokenized
-	 * 
+	 *
 	 * @return the resulting tokens
 	 */
+	@Override
 	protected List<String> doTokenize(String line) {
 
 		List<String> tokens = new ArrayList<String>();
@@ -147,12 +149,17 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
 			char currentChar = chars[i];
 			boolean isEnd = (i == (length - 1));
 
-			if ((isDelimiterCharacter(currentChar) && !inQuoted) || isEnd) {
+			boolean isDelimiter = isDelimiter(chars, i, delimiter);
+
+			if ((isDelimiter && !inQuoted) || isEnd) {
 				int endPosition = (isEnd ? (length - lastCut) : (i - lastCut));
 
-				if (isEnd && isDelimiterCharacter(currentChar)) {
+				if (isEnd && isDelimiter) {
 					endPosition--;
 				}
+				else if (!isEnd){
+					endPosition = (endPosition - delimiter.length()) + 1;
+				}
 
 				if (includedFields == null || includedFields.contains(fieldCount)) {
 					String value = maybeStripQuotes(new String(chars, lastCut, endPosition));
@@ -161,7 +168,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
 
 				fieldCount++;
 
-				if (isEnd && (isDelimiterCharacter(currentChar))) {
+				if (isEnd && (isDelimiter)) {
 					if (includedFields == null || includedFields.contains(fieldCount)) {
 						tokens.add("");
 					}
@@ -183,7 +190,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
 	 * If the string is quoted strip (possibly with whitespace outside the
 	 * quotes (which will be stripped), replace escaped quotes inside the
 	 * string. Quotes are escaped with double instances of the quote character.
-	 * 
+	 *
 	 * @param string
 	 * @return the same string but stripped and unescaped if necessary
 	 */
@@ -204,7 +211,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
 
 	/**
 	 * Is this string surrounded by quote characters?
-	 * 
+	 *
 	 * @param value
 	 * @return true if the value starts and ends with the
 	 * {@link #quoteCharacter}
@@ -218,19 +225,28 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
 
 	/**
 	 * Is the supplied character the delimiter character?
-	 * 
+	 *
 	 * @param c the character to be checked
 	 * @return <code>true</code> if the supplied character is the delimiter
 	 * character
 	 * @see DelimitedLineTokenizer#DelimitedLineTokenizer(char)
 	 */
-	private boolean isDelimiterCharacter(char c) {
-		return c == this.delimiter;
+	private boolean isDelimiter(char[] chars, int i, String token) {
+		boolean result = false;
+
+		if(i >= token.length()) {
+			String end = new String(chars, (i-token.length()) + 1, token.length());
+			if(token.equals(end)) {
+				result = true;
+			}
+		}
+
+		return result;
 	}
 
 	/**
 	 * Is the supplied character a quote character?
-	 * 
+	 *
 	 * @param c the character to be checked
 	 * @return <code>true</code> if the supplied character is an quote character
 	 * @see #setQuoteCharacter(char)