Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

BATCH-1696: don't trim whitespace from unquoted text

Browse Source
This commit is contained in:
Dave Syer
2011-03-15 09:29:21 +00:00
parent a2e831cf6d
commit 57b0cb7498
2 changed files with 40 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java
View File

@@ -168,8 +168,9 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer {
endLength = 1;
}
value = value.substring(1, endLength);
return value;
}
return value;
return string;
}
/**

41
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java
View File

@@ -16,15 +16,20 @@
package org.springframework.batch.item.file.transform;
import junit.framework.TestCase;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import org.junit.Test;
public class DelimitedLineTokenizerTests extends TestCase {
public class DelimitedLineTokenizerTests {
private static final String TOKEN_MATCHES = "token equals the expected string";
private DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
@Test
public void testTokenizeRegularUse() {
FieldSet tokens = tokenizer.tokenize("sfd,\"Well,I have no idea what to do in the afternoon\",sFj, asdf,,as\n");
assertEquals(6, tokens.getFieldCount());
@@ -41,6 +46,7 @@ public class DelimitedLineTokenizerTests extends TestCase {
assertTrue(TOKEN_MATCHES, tokens.readString(1).equals(""));
}
@Test
public void testInvalidConstructorArgument() {
try {
new DelimitedLineTokenizer(DelimitedLineTokenizer.DEFAULT_QUOTE_CHARACTER);
@@ -51,11 +57,13 @@ public class DelimitedLineTokenizerTests extends TestCase {
}
}
@Test
public void testDelimitedLineTokenizer() {
FieldSet line = tokenizer.tokenize("a,b,c");
assertEquals(3, line.getFieldCount());
}
@Test
public void testNames() {
tokenizer.setNames(new String[] {"A", "B", "C"});
FieldSet line = tokenizer.tokenize("a,b,c");
@@ -63,6 +71,7 @@ public class DelimitedLineTokenizerTests extends TestCase {
assertEquals("a", line.readString("A"));
}
@Test
public void testTooFewNames() {
tokenizer.setNames(new String[] {"A", "B"});
try {
@@ -75,6 +84,7 @@ public class DelimitedLineTokenizerTests extends TestCase {
}
}
@Test
public void testTooFewNamesNotStrict() {
tokenizer.setNames(new String[] {"A", "B"});
tokenizer.setStrict(false);
@@ -85,6 +95,7 @@ public class DelimitedLineTokenizerTests extends TestCase {
assertTrue(TOKEN_MATCHES, tokens.readString(1).equals("b"));
}
@Test
public void testTooManyNames() {
tokenizer.setNames(new String[] {"A", "B", "C", "D"});
try{
@@ -97,6 +108,7 @@ public class DelimitedLineTokenizerTests extends TestCase {
}
@Test
public void testTooManyNamesNotStrict() {
tokenizer.setNames(new String[] {"A", "B", "C", "D","E"});
tokenizer.setStrict( false );
@@ -110,18 +122,21 @@ public class DelimitedLineTokenizerTests extends TestCase {
assertTrue(TOKEN_MATCHES, tokens.readString(4).equals(""));
}
@Test
public void testDelimitedLineTokenizerChar() {
AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(' ');
FieldSet line = tokenizer.tokenize("a b c");
assertEquals(3, line.getFieldCount());
}
@Test
public void testTokenizeWithQuotes() {
FieldSet line = tokenizer.tokenize("a,b,\"c\"");
assertEquals(3, line.getFieldCount());
assertEquals("c", line.readString(2));
}
@Test
public void testTokenizeWithNotDefaultQuotes() {
tokenizer.setQuoteCharacter('\'');
FieldSet line = tokenizer.tokenize("a,b,'c'");
@@ -129,6 +144,7 @@ public class DelimitedLineTokenizerTests extends TestCase {
assertEquals("c", line.readString(2));
}
@Test
public void testTokenizeWithEscapedQuotes() {
FieldSet line = tokenizer.tokenize("a,\"\"b,\"\"\"c\"");
assertEquals(3, line.getFieldCount());
@@ -136,6 +152,7 @@ public class DelimitedLineTokenizerTests extends TestCase {
assertEquals("\"c", line.readString(2));
}
@Test
public void testTokenizeWithUnclosedQuotes() {
tokenizer.setQuoteCharacter('\'');
FieldSet line = tokenizer.tokenize("a,\"b,c");
@@ -144,24 +161,35 @@ public class DelimitedLineTokenizerTests extends TestCase {
assertEquals("c", line.readString(2));
}
@Test
public void testTokenizeWithSpaceInField() {
FieldSet line = tokenizer.tokenize("a,b ,c");
assertEquals(3, line.getFieldCount());
assertEquals("b ", line.readRawString(1));
}
@Test
public void testTokenizeWithSpaceAtEnd() {
FieldSet line = tokenizer.tokenize("a,b,c ");
assertEquals(3, line.getFieldCount());
assertEquals("c", line.readString(2));
assertEquals("c ", line.readRawString(2));
}
@Test
public void testTokenizeWithQuoteAndSpaceAtEnd() {
FieldSet line = tokenizer.tokenize("a,b,\"c\" ");
assertEquals(3, line.getFieldCount());
assertEquals("c", line.readString(2));
}
@Test
public void testTokenizeWithQuoteAndSpaceBeforeDelimiter() {
FieldSet line = tokenizer.tokenize("a,\"b\" ,c");
assertEquals(3, line.getFieldCount());
assertEquals("b", line.readString(1));
}
@Test
public void testTokenizeWithDelimiterAtEnd() {
FieldSet line = tokenizer.tokenize("a,b,c,");
assertEquals(4, line.getFieldCount());
@@ -169,11 +197,13 @@ public class DelimitedLineTokenizerTests extends TestCase {
assertEquals("", line.readString(3));
}
@Test
public void testEmptyLine() throws Exception {
FieldSet line = tokenizer.tokenize("");
assertEquals(0, line.getFieldCount());
}
@Test
public void testEmptyLineWithNames(){
tokenizer.setNames(new String[]{"A", "B"});
@@ -186,18 +216,21 @@ public class DelimitedLineTokenizerTests extends TestCase {
}
}
@Test
public void testWhitespaceLine() throws Exception {
FieldSet line = tokenizer.tokenize(" ");
// whitespace counts as text
assertEquals(1, line.getFieldCount());
}
@Test
public void testNullLine() throws Exception {
FieldSet line = tokenizer.tokenize(null);
// null doesn't...
assertEquals(0, line.getFieldCount());
}
@Test
public void testMultiLineField() throws Exception {
FieldSet line = tokenizer.tokenize("a,b,c\nrap");
assertEquals(3, line.getFieldCount());
@@ -205,6 +238,7 @@ public class DelimitedLineTokenizerTests extends TestCase {
}
@Test
public void testMultiLineFieldWithQuotes() throws Exception {
FieldSet line = tokenizer.tokenize("a,b,\"c\nrap\"");
assertEquals(3, line.getFieldCount());
@@ -212,6 +246,7 @@ public class DelimitedLineTokenizerTests extends TestCase {
}
@Test
public void testTokenizeWithQuotesEmptyValue() {
FieldSet line = tokenizer.tokenize("\"a\",\"b\",\"\",\"d\"");
assertEquals(4, line.getFieldCount());