Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

OPEN - issue BATCH-809: fixedLenghtTokenizer problem with 1.1.2

Browse Source 
Add strict flag and open-ended last range logic
This commit is contained in:
dsyer
2008-09-04 08:15:47 +00:00
parent 03aa49f09d
commit 4bc943537d
4 changed files with 133 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/FixedLengthTokenizerTests.java
View File

@@ -29,40 +29,40 @@ public class FixedLengthTokenizerTests {
private String line = null;
/**
* if null or empty string is tokenized, tokenizer returns empty fieldset
* if null or empty string is tokenized, tokenizer returns empty fieldset
* (with no tokens).
*/
@Test
public void testTokenizeEmptyString() {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)});
try{
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10), new Range(11, 15) });
try {
tokenizer.tokenize("");
fail("Expected IncorrectLineLengthException");
}
catch(IncorrectLineLengthException ex){
catch (IncorrectLineLengthException ex) {
assertEquals(15, ex.getExpectedLength());
assertEquals(0, ex.getActualLength());
}
}
@Test
public void testEmptyStringWithNoRanges(){
tokenizer.setColumns(new Range[]{});
public void testEmptyStringWithNoRanges() {
tokenizer.setColumns(new Range[] {});
tokenizer.tokenize("");
}
@Test
public void testTokenizeSmallerStringThanRanges() {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)});
try{
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10), new Range(11, 15) });
try {
tokenizer.tokenize("12345");
fail("Expected IncorrectLineLengthException");
}
catch(IncorrectLineLengthException ex){
catch (IncorrectLineLengthException ex) {
assertEquals(15, ex.getExpectedLength());
assertEquals(5, ex.getActualLength());
}
}
@Test
@@ -73,19 +73,37 @@ public class FixedLengthTokenizerTests {
assertEquals("", tokens.readString(1));
}
@Test
public void testTokenizeSmallerStringThanRangesNotStrict() {
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10) });
tokenizer.setStrict(false);
FieldSet tokens = tokenizer.tokenize("12345");
assertEquals("12345", tokens.readString(0));
assertEquals("", tokens.readString(1));
}
@Test
public void testTokenizeSmallerStringThanRangesWithWhitespaceOpenEnded() {
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6) });
FieldSet tokens = tokenizer.tokenize("12345 ");
assertEquals("12345", tokens.readString(0));
assertEquals("", tokens.readString(1));
}
@Test
public void testTokenizeNullString() {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,10),new Range(11,15)});
try{
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10), new Range(11, 15) });
try {
tokenizer.tokenize(null);
fail("Expected IncorrectLineLengthException");
}
catch(IncorrectLineLengthException ex){}
catch (IncorrectLineLengthException ex) {
}
}
@Test
public void testTokenizeRegularUse() {
tokenizer.setColumns(new Range[] {new Range(1,2),new Range(3,7),new Range(8,12)});
tokenizer.setColumns(new Range[] { new Range(1, 2), new Range(3, 7), new Range(8, 12) });
// test shorter line as defined by record descriptor
line = "H11234512345";
FieldSet tokens = tokenizer.tokenize(line);
@@ -94,48 +112,69 @@ public class FixedLengthTokenizerTests {
assertEquals("12345", tokens.readString(1));
assertEquals("12345", tokens.readString(2));
}
@Test
public void testNormalLength() throws Exception {
tokenizer.setColumns(new Range[] {new Range(1,10),new Range(11,25),new Range(26,30)});
tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26, 30) });
// test shorter line as defined by record descriptor
line = "H1 12345678 12345";
FieldSet tokens = tokenizer.tokenize(line);
assertEquals(3, tokens.getFieldCount());
assertEquals(line.substring(0, 10).trim(), tokens.readString(0));
assertEquals(line.substring(10, 25).trim(), tokens.readString(1));
assertEquals(line.substring(25).trim(), tokens.readString(2));
assertEquals(line.substring(25).trim(), tokens.readString(2));
}
@Test
public void testLongerLines() throws Exception {
tokenizer.setColumns(new Range[] {new Range(1,10),new Range(11,25),new Range(26,30)});
tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26, 30) });
line = "H1 12345678 1234567890";
try{
try {
tokenizer.tokenize(line);
fail("Expected IncorrectLineLengthException");
}
catch(IncorrectLineLengthException ex){
catch (IncorrectLineLengthException ex) {
assertEquals(30, ex.getExpectedLength());
assertEquals(35, ex.getActualLength());
}
}
@Test
public void testLongerLinesOpenRange() throws Exception {
tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26) });
line = "H1 12345678 1234567890";
FieldSet tokens = tokenizer.tokenize(line);
assertEquals(line.substring(0, 10).trim(), tokens.readString(0));
assertEquals(line.substring(10, 25).trim(), tokens.readString(1));
assertEquals(line.substring(25).trim(), tokens.readString(2));
}
@Test
public void testLongerLinesNotStrict() throws Exception {
tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26,30) });
line = "H1 12345678 1234567890";
tokenizer.setStrict(false);
FieldSet tokens = tokenizer.tokenize(line);
assertEquals(line.substring(0, 10).trim(), tokens.readString(0));
assertEquals(line.substring(10, 25).trim(), tokens.readString(1));
assertEquals(line.substring(25, 30).trim(), tokens.readString(2));
}
@Test
public void testNonAdjacentRangesUnsorted() throws Exception {
tokenizer.setColumns(new Range[] {new Range(14,28), new Range(34,38), new Range(1,10)});
tokenizer.setColumns(new Range[] { new Range(14, 28), new Range(34, 38), new Range(1, 10) });
// test normal length
line = "H1 +++12345678 +++++12345";
FieldSet tokens = tokenizer.tokenize(line);
assertEquals(3, tokens.getFieldCount());
assertEquals(line.substring(0, 10).trim(), tokens.readString(2));
assertEquals(line.substring(13, 28).trim(), tokens.readString(0));
assertEquals(line.substring(33, 38).trim(), tokens.readString(1));
assertEquals(line.substring(33, 38).trim(), tokens.readString(1));
}
@Test
public void testAnotherTypeOfRecord() throws Exception {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,15),new Range(16,25),new Range(26,27)});
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 15), new Range(16, 25), new Range(26, 27) });
// test another type of record
line = "H2 123456 12345 12";
FieldSet tokens = tokenizer.tokenize(line);
@@ -143,12 +182,13 @@ public class FixedLengthTokenizerTests {
assertEquals(line.substring(0, 5).trim(), tokens.readString(0));
assertEquals(line.substring(5, 15).trim(), tokens.readString(1));
assertEquals(line.substring(15, 25).trim(), tokens.readString(2));
assertEquals(line.substring(25).trim(), tokens.readString(3));
assertEquals(line.substring(25).trim(), tokens.readString(3));
}
@Test
public void testFillerAtEnd() throws Exception {
tokenizer.setColumns(new Range[] {new Range(1,5),new Range(6,15),new Range(16,25),new Range(26,27),new Range(34)});
tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 15), new Range(16, 25), new Range(26, 27),
new Range(34) });
// test another type of record
line = "H2 123456 12345 12-123456";
FieldSet tokens = tokenizer.tokenize(line);
@@ -156,13 +196,13 @@ public class FixedLengthTokenizerTests {
assertEquals(line.substring(0, 5).trim(), tokens.readString(0));
assertEquals(line.substring(5, 15).trim(), tokens.readString(1));
assertEquals(line.substring(15, 25).trim(), tokens.readString(2));
assertEquals(line.substring(25, 27).trim(), tokens.readString(3));
assertEquals(line.substring(25, 27).trim(), tokens.readString(3));
}
@Test
public void testTokenizerInvalidSetup() {
tokenizer.setNames(new String[] {"a", "b"});
tokenizer.setColumns(new Range[] {new Range(1,5)});
tokenizer.setNames(new String[] { "a", "b" });
tokenizer.setColumns(new Range[] { new Range(1, 5) });
try {
tokenizer.tokenize("12345");