From 6ba7d6936d44160f0f4dc6d98bd15dcef932f492 Mon Sep 17 00:00:00 2001 From: lucasward Date: Thu, 9 Oct 2008 22:42:05 +0000 Subject: [PATCH] OPEN - issue BATCH-607: FlatFileItemWriter section needs additional details http://jira.springframework.org/browse/BATCH-607 Added a section on FlatFile exceptions. --- .../docbook/reference/readersAndWriters.xml | 114 +++++++++++++++++- 1 file changed, 108 insertions(+), 6 deletions(-) diff --git a/docs/src/site/docbook/reference/readersAndWriters.xml b/docs/src/site/docbook/reference/readersAndWriters.xml index 84a9561cb..9a8d97127 100644 --- a/docs/src/site/docbook/reference/readersAndWriters.xml +++ b/docs/src/site/docbook/reference/readersAndWriters.xml @@ -334,9 +334,9 @@ LineTokenizer to translate a line of data from a resource into an object of the desired type: - public interface FieldSetMapper { + public interface FieldSetMapper<T> { - public Object mapLine(FieldSet fs); + T mapFieldSet(FieldSet fieldSet); } @@ -422,7 +422,7 @@ if (line != null) { FieldSet tokenizedLine = tokenizer.tokenize(line); - return fieldSetMapper.mapLine(tokenizedLine); + return fieldSetMapper.mapFieldSet(tokenizedLine); } return null; @@ -620,7 +620,7 @@ <bean id="fixedLengthLineTokenizer" - class="org.springframework.batch.item.file.transform.FixedLengthTokenizer"> + class="org.springframework.batch.io.file.transform.FixedLengthTokenizer"> <property name="names" value="ISIN, Quantity, Price, Customer" /> <property name="columns" value="1-12, 13-15, 16-20, 21-29" /> </bean> @@ -676,7 +676,7 @@ prefix in a line with a particular tokenizer: <bean id="orderFileDescriptor" - class="org.springframework.batch.item.file.transform.PrefixMatchingCompositeLineTokenizer"> + class="org.springframework.batch.io.file.transform.PrefixMatchingCompositeLineTokenizer"> <property name="tokenizers"> <map> <entry key="HEA" value-ref="headerRecordDescriptor" /> @@ -700,6 +700,108 @@ the record is returned, allowing them to return a complete order as one 'item'. + +
+ Exception Handling in flat files + + There are many scenarios when tokenizing a line that cause + exceptions to be thrown. Many flat files are imperfect and contain + records that aren't formatted correctly. Many users choose to skip the + lines causing these errors, logging out the issue, original line, and + line number, for manual inspection later. (or by another batch job) + For this reason, Spring Batch provides a hierarchy of exceptions for + handling parse exceptions: + FlatFileParseException and + FlatFileFormatException. + FlatFileParseException is thrown by the + FlatFileItemReader when any errors are + encountered while trying to read a file. + FlatFileFormatException is thrown by + implementations of the LineTokenizer interface, + and indicates a more specific error encountered while + tokenizing. + +
+ IncorrectTokenCountException + + Both DelimitedLineTokenizer and + FixedLengthLineTokenizer have the ability to + specify column names that can be used for creating a + FieldSet. However, if the number of column + names doesn't match the number of columns found while tokenizing a + line the FieldSet can't be created, and a + IncorrectTokenCountException is thrown, which contains the number of + tokens encountered, and the number expected: + + + tokenizer.setNames(new String[] {"A", "B", "C", "D"}); + + try{ + tokenizer.tokenize("a,b,c"); + } + catch(IncorrectTokenCountException e){ + assertEquals(4, e.getExpectedCount()); + assertEquals(3, e.getActualCount()); + } + + + + Because the tokenizer was configured with 4 columns, but only + 3 tokens were found in the file, an IncorrectTokenCountException was + thrown. +
+ +
+ IncorrectLineLengthException + + Files formatted in a fixed length format have additional + requirements when parsing because unlike a delimited format, each + column must strictly adhere to the width defined for it. If the + total line length doesn't add up to the widest value of this column, + an exception is thrown: + + + tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10), new Range(11, 15) }); + try { + tokenizer.tokenize("12345"); + fail("Expected IncorrectLineLengthException"); + } + catch (IncorrectLineLengthException ex) { + assertEquals(15, ex.getExpectedLength()); + assertEquals(5, ex.getActualLength()); + } + + + + The configured ranges for the tokenizer above are: 1-5, 6-10, + and 11-15, thus the total length of the line expected is 15. + However, in this case a line of length 5 was passed in, causing an + IncorrectLineLengthException to be thrown. + Throwing an exception here rather than only mapping the first column + allows the processing of the line to fail earlier, and with more + information than it would if it failed while trying to read in + column 2 in a FieldSetMapper. However, there + are scenarios where the length of the line isn't always constant. + For this reason, validation of line length can be turned off via the + 'strict' property: + + + tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 10) }); + tokenizer.setStrict(false); + FieldSet tokens = tokenizer.tokenize("12345"); + assertEquals("12345", tokens.readString(0)); + assertEquals("", tokens.readString(1)); + + + + The above example is almost identical to the one before it, + except the tokenizer.setStrict(false) was called. This setting tells + the tokenizer to not enforce line lengths when tokenizing the line. + A FieldSet is now correctly created and + returned. However, it will only contain empty tokens for the + remaining values. +
+
@@ -768,7 +870,7 @@ would look like the following: <bean id="itemWriter" - class="org.springframework.batch.item.file.FlatFileItemWriter"> + class="org.springframework.batch.io.file.FlatFileItemWriter"> <property name="resource" value="file:target/test-outputs/20070122.testStream.multilineStep.txt" /> <property name="lineAggregator">