From 1b71ada1ff1015f2bf82b2d741ef8b7fa8613c0a Mon Sep 17 00:00:00 2001 From: Michael Minella Date: Thu, 11 Oct 2012 12:22:54 -0500 Subject: [PATCH] BATCH-1869: Updated DelimitedLineTokenizer to support String delimiters --- .../transform/DelimitedLineTokenizer.java | 68 ++++++++++++------- .../DelimitedLineTokenizerTests.java | 59 +++++++++++----- 2 files changed, 85 insertions(+), 42 deletions(-) diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java index d94dc08f5..fd53c4eb2 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java @@ -1,5 +1,5 @@ /* - * Copyright 2006-2007 the original author or authors. + * Copyright 2006-2012 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,21 +28,22 @@ import org.springframework.util.StringUtils; * A {@link LineTokenizer} implementation that splits the input String on a * configurable delimiter. This implementation also supports the use of an * escape character to escape delimiters and line endings. - * + * * @author Rob Harrop * @author Dave Syer - * + * @author Michael Minella */ +@Deprecated public class DelimitedLineTokenizer extends AbstractLineTokenizer { /** * Convenient constant for the common case of a tab delimiter. */ - public static final char DELIMITER_TAB = '\t'; + public static final String DELIMITER_TAB = "\t"; /** * Convenient constant for the common case of a comma delimiter. */ - public static final char DELIMITER_COMMA = ','; + public static final String DELIMITER_COMMA = ","; /** * Convenient constant for the common case of a " character used to escape @@ -51,7 +52,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { public static final char DEFAULT_QUOTE_CHARACTER = '"'; // the delimiter character used when reading input. - private char delimiter; + private String delimiter; private char quoteCharacter = DEFAULT_QUOTE_CHARACTER; @@ -62,7 +63,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { /** * Create a new instance of the {@link DelimitedLineTokenizer} class for the * common case where the delimiter is a {@link #DELIMITER_COMMA comma}. - * + * * @see #DelimitedLineTokenizer(char) * @see #DELIMITER_COMMA */ @@ -72,11 +73,11 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { /** * Create a new instance of the {@link DelimitedLineTokenizer} class. - * + * * @param delimiter the desired delimiter */ - public DelimitedLineTokenizer(char delimiter) { - Assert.state(delimiter != DEFAULT_QUOTE_CHARACTER, "[" + DEFAULT_QUOTE_CHARACTER + public DelimitedLineTokenizer(String delimiter) { + Assert.state(!delimiter.equals(String.valueOf(DEFAULT_QUOTE_CHARACTER)), "[" + DEFAULT_QUOTE_CHARACTER + "] is not allowed as delimiter for tokenizers."); this.delimiter = delimiter; @@ -85,10 +86,10 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { /** * Setter for the delimiter character. - * + * * @param delimiter */ - public void setDelimiter(char delimiter) { + public void setDelimiter(String delimiter) { this.delimiter = delimiter; } @@ -97,7 +98,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { * default all fields are included, but this property can be set to pick out * only a few fields from a larger set. Note that if field names are * provided, their number must match the number of included fields. - * + * * @param includedFields the included fields to set */ public void setIncludedFields(int[] includedFields) { @@ -112,9 +113,9 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { * extend a field across line endings or to enclose a String which contains * the delimiter. Inside a quoted token the quote character can be used to * escape itself, thus "a""b""c" is tokenized to a"b"c. - * + * * @param quoteCharacter the quoteCharacter to set - * + * * @see #DEFAULT_QUOTE_CHARACTER */ public final void setQuoteCharacter(char quoteCharacter) { @@ -125,11 +126,12 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { /** * Yields the tokens resulting from the splitting of the supplied * line. - * + * * @param line the line to be tokenized - * + * * @return the resulting tokens */ + @Override protected List doTokenize(String line) { List tokens = new ArrayList(); @@ -147,12 +149,17 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { char currentChar = chars[i]; boolean isEnd = (i == (length - 1)); - if ((isDelimiterCharacter(currentChar) && !inQuoted) || isEnd) { + boolean isDelimiter = isDelimiter(chars, i, delimiter); + + if ((isDelimiter && !inQuoted) || isEnd) { int endPosition = (isEnd ? (length - lastCut) : (i - lastCut)); - if (isEnd && isDelimiterCharacter(currentChar)) { + if (isEnd && isDelimiter) { endPosition--; } + else if (!isEnd){ + endPosition = (endPosition - delimiter.length()) + 1; + } if (includedFields == null || includedFields.contains(fieldCount)) { String value = maybeStripQuotes(new String(chars, lastCut, endPosition)); @@ -161,7 +168,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { fieldCount++; - if (isEnd && (isDelimiterCharacter(currentChar))) { + if (isEnd && (isDelimiter)) { if (includedFields == null || includedFields.contains(fieldCount)) { tokens.add(""); } @@ -183,7 +190,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { * If the string is quoted strip (possibly with whitespace outside the * quotes (which will be stripped), replace escaped quotes inside the * string. Quotes are escaped with double instances of the quote character. - * + * * @param string * @return the same string but stripped and unescaped if necessary */ @@ -204,7 +211,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { /** * Is this string surrounded by quote characters? - * + * * @param value * @return true if the value starts and ends with the * {@link #quoteCharacter} @@ -218,19 +225,28 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer { /** * Is the supplied character the delimiter character? - * + * * @param c the character to be checked * @return true if the supplied character is the delimiter * character * @see DelimitedLineTokenizer#DelimitedLineTokenizer(char) */ - private boolean isDelimiterCharacter(char c) { - return c == this.delimiter; + private boolean isDelimiter(char[] chars, int i, String token) { + boolean result = false; + + if(i >= token.length()) { + String end = new String(chars, (i-token.length()) + 1, token.length()); + if(token.equals(end)) { + result = true; + } + } + + return result; } /** * Is the supplied character a quote character? - * + * * @param c the character to be checked * @return true if the supplied character is an quote character * @see #setQuoteCharacter(char) diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java index 9ec4d7ba9..ab1e7093a 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2006-2007 the original author or authors. + * Copyright 2006-2012 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,7 +49,7 @@ public class DelimitedLineTokenizerTests { @Test public void testInvalidConstructorArgument() { try { - new DelimitedLineTokenizer(DelimitedLineTokenizer.DEFAULT_QUOTE_CHARACTER); + new DelimitedLineTokenizer(String.valueOf(DelimitedLineTokenizer.DEFAULT_QUOTE_CHARACTER)); fail("Quote character can't be used as delimiter for delimited line tokenizer!"); } catch (Exception e) { @@ -83,18 +83,18 @@ public class DelimitedLineTokenizerTests { assertEquals(3, e.getActualCount()); } } - + @Test public void testTooFewNamesNotStrict() { tokenizer.setNames(new String[] {"A", "B"}); tokenizer.setStrict(false); FieldSet tokens = tokenizer.tokenize("a,b,c"); - + assertTrue(TOKEN_MATCHES, tokens.readString(0).equals("a")); assertTrue(TOKEN_MATCHES, tokens.readString(1).equals("b")); - } - + } + @Test public void testTooManyNames() { tokenizer.setNames(new String[] {"A", "B", "C", "D"}); @@ -105,30 +105,57 @@ public class DelimitedLineTokenizerTests { assertEquals(4, e.getExpectedCount()); assertEquals(3, e.getActualCount()); } - + } - + @Test public void testTooManyNamesNotStrict() { tokenizer.setNames(new String[] {"A", "B", "C", "D","E"}); tokenizer.setStrict( false ); FieldSet tokens = tokenizer.tokenize("a,b,c"); - + assertTrue(TOKEN_MATCHES, tokens.readString(0).equals("a")); assertTrue(TOKEN_MATCHES, tokens.readString(1).equals("b")); assertTrue(TOKEN_MATCHES, tokens.readString(2).equals("c")); assertTrue(TOKEN_MATCHES, tokens.readString(3).equals("")); - assertTrue(TOKEN_MATCHES, tokens.readString(4).equals("")); - } + assertTrue(TOKEN_MATCHES, tokens.readString(4).equals("")); + } @Test public void testDelimitedLineTokenizerChar() { - AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(' '); + AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" "); FieldSet line = tokenizer.tokenize("a b c"); assertEquals(3, line.getFieldCount()); } + @Test + public void testDelimitedLineTokenizerString() { + AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" b "); + FieldSet line = tokenizer.tokenize("a b c"); + assertEquals(2, line.getFieldCount()); + assertEquals("a", line.readString(0)); + assertEquals("c", line.readString(1)); + } + + @Test + public void testDelimitedLineTokenizerNewlineToken() { + AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer("\n"); + FieldSet line = tokenizer.tokenize("a b\n c"); + assertEquals(2, line.getFieldCount()); + assertEquals("a b", line.readString(0)); + assertEquals("c", line.readString(1)); + } + + @Test + public void testDelimitedLineTokenizerWrappedToken() { + AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer("\nrap"); + FieldSet line = tokenizer.tokenize("a b\nrap c"); + assertEquals(2, line.getFieldCount()); + assertEquals("a b", line.readString(0)); + assertEquals("c", line.readString(1)); + } + @Test public void testTokenizeWithQuotes() { FieldSet line = tokenizer.tokenize("a,b,\"c\""); @@ -202,10 +229,10 @@ public class DelimitedLineTokenizerTests { FieldSet line = tokenizer.tokenize(""); assertEquals(0, line.getFieldCount()); } - + @Test public void testEmptyLineWithNames(){ - + tokenizer.setNames(new String[]{"A", "B"}); try{ tokenizer.tokenize(""); @@ -245,7 +272,7 @@ public class DelimitedLineTokenizerTests { assertEquals("c\nrap", line.readString(2)); } - + @Test public void testTokenizeWithQuotesEmptyValue() { FieldSet line = tokenizer.tokenize("\"a\",\"b\",\"\",\"d\""); @@ -296,4 +323,4 @@ public class DelimitedLineTokenizerTests { assertEquals("c", line.readString("bar")); } -} +} \ No newline at end of file