diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/BufferedReaderFactory.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/BufferedReaderFactory.java new file mode 100644 index 000000000..194a6aec3 --- /dev/null +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/BufferedReaderFactory.java @@ -0,0 +1,48 @@ +/* + * Copyright 2006-2007 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.item.file; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.UnsupportedEncodingException; + +import org.springframework.core.io.Resource; + +/** + * A factory strategy for custom extensions of {@link BufferedReader} allowing + * customisation of the standard behaviour of the java.io variety. + * + * @author Dave Syer + * + * @since 2.1 + */ +public interface BufferedReaderFactory { + + /** + * Create a {@link BufferedReader} for reading String items from the + * provided resource. + * + * @param resource a {@link Resource} containing the data to be read + * @param encoding the encoding required for converting binary data to + * String + * @return a {@link BufferedReader} + * @throws UnsupportedEncodingException if the encoding is not supported by + * the platform + * @throws IOException if there is a problem creating the reader + */ + BufferedReader create(Resource resource, String encoding) throws UnsupportedEncodingException, IOException; + +} diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/DefaultBufferedReaderFactory.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/DefaultBufferedReaderFactory.java new file mode 100644 index 000000000..91d8f7fde --- /dev/null +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/DefaultBufferedReaderFactory.java @@ -0,0 +1,36 @@ +/* + * Copyright 2006-2007 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.item.file; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; + +import org.springframework.core.io.Resource; + +/** + * @author Dave Syer + * + * @since 2.1 + */ +public class DefaultBufferedReaderFactory implements BufferedReaderFactory { + + public BufferedReader create(Resource resource, String encoding) throws UnsupportedEncodingException, IOException { + return new BufferedReader(new InputStreamReader(resource.getInputStream(), encoding)); + } + +} diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/FlatFileItemReader.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/FlatFileItemReader.java index f8208d72b..3698ab95d 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/FlatFileItemReader.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/FlatFileItemReader.java @@ -18,7 +18,6 @@ package org.springframework.batch.item.file; import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; import java.nio.charset.Charset; import org.apache.commons.logging.Log; @@ -71,6 +70,8 @@ public class FlatFileItemReader extends AbstractItemCountingItemStreamItemRea private boolean strict = true; + private BufferedReaderFactory bufferedReaderFactory = new DefaultBufferedReaderFactory(); + public FlatFileItemReader() { setName(ClassUtils.getShortName(FlatFileItemReader.class)); } @@ -84,7 +85,7 @@ public class FlatFileItemReader extends AbstractItemCountingItemStreamItemRea public void setStrict(boolean strict) { this.strict = strict; } - + /** * @param skippedLinesCallback will be called for each one of the initial * skipped lines before any items are read. @@ -124,6 +125,18 @@ public class FlatFileItemReader extends AbstractItemCountingItemStreamItemRea this.encoding = encoding; } + /** + * Factory for the {@link BufferedReader} that will be used to extract lines + * from the file. The default is fine for plain text files, but this is a + * useful strategy for binary files where the standard BufferedReaader from + * java.io is limiting. + * + * @param bufferedReaderFactory the bufferedReaderFactory to set + */ + public void setBufferedReaderFactory(BufferedReaderFactory bufferedReaderFactory) { + this.bufferedReaderFactory = bufferedReaderFactory; + } + /** * Setter for comment prefixes. Can be used to ignore header lines as well * by using e.g. the first couple of column names as a prefix. @@ -175,12 +188,12 @@ public class FlatFileItemReader extends AbstractItemCountingItemStreamItemRea return null; } else { - try{ + try { return lineMapper.mapLine(logicalLine, lineCount); } - catch(Exception ex){ - logger.error("Parsing error at line: " + lineCount + " in resource=" + - resource.getDescription() + ", input=[" + line + "]", ex); + catch (Exception ex) { + logger.error("Parsing error at line: " + lineCount + " in resource=" + resource.getDescription() + + ", input=[" + line + "]", ex); throw ex; } } @@ -229,7 +242,7 @@ public class FlatFileItemReader extends AbstractItemCountingItemStreamItemRea @Override protected void doClose() throws Exception { lineCount = 0; - if (reader!=null) { + if (reader != null) { reader.close(); } } @@ -242,14 +255,14 @@ public class FlatFileItemReader extends AbstractItemCountingItemStreamItemRea noInput = false; if (!resource.exists()) { if (strict) { - throw new IllegalStateException("Input resource must exist (reader is in 'strict' mode): "+resource); + throw new IllegalStateException("Input resource must exist (reader is in 'strict' mode): " + resource); } noInput = true; logger.warn("Input resource does not exist " + resource.getDescription()); return; } - reader = new BufferedReader(new InputStreamReader(resource.getInputStream(), encoding)); + reader = bufferedReaderFactory.create(resource, encoding); for (int i = 0; i < linesToSkip; i++) { String line = readLine(); if (skippedLinesCallback != null) { diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/SimpleBinaryBufferedReaderFactory.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/SimpleBinaryBufferedReaderFactory.java new file mode 100644 index 000000000..87d2af66e --- /dev/null +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/SimpleBinaryBufferedReaderFactory.java @@ -0,0 +1,143 @@ +/* + * Copyright 2006-2007 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.item.file; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.UnsupportedEncodingException; + +import org.springframework.core.io.Resource; + +/** + * A {@link BufferedReaderFactory} useful for reading simple binary (or text) + * files with no line endings, such as those produced by mainframe copy books. + * The reader splits a stream up across fixed line endings (rather than the + * usual convention based on plain text). The line endings are discarded, just + * as with the default plain text implementation. + * + * @author Dave Syer + * + * @since 2.1 + */ +public class SimpleBinaryBufferedReaderFactory implements BufferedReaderFactory { + + /** + * The default line ending value. + */ + private static final String DEFAULT_LINE_ENDING = "\n"; + + private String lineEnding = DEFAULT_LINE_ENDING; + + /** + * @param lineEnding + */ + public void setLineEnding(String lineEnding) { + this.lineEnding = lineEnding; + } + + public BufferedReader create(Resource resource, String encoding) throws UnsupportedEncodingException, IOException { + return new BinaryBufferedReader(new InputStreamReader(resource.getInputStream(), encoding), lineEnding); + } + + /** + * BufferedReader extension that splits lines based on a line ending, rather + * than the usual plain text conventions. + * + * @author Dave Syer + * + */ + private final class BinaryBufferedReader extends BufferedReader { + + private final String ending; + + /** + * @param in + */ + private BinaryBufferedReader(Reader in, String ending) { + super(in); + this.ending = ending; + } + + @Override + public String readLine() throws IOException { + + StringBuilder buffer = null; + + synchronized (lock) { + + int next = read(); + if (next == -1) { + return null; + } + + buffer = new StringBuilder(); + StringBuilder candidateEnding = new StringBuilder(); + + while (!isEndOfLine(buffer, candidateEnding, next)) { + next = read(); + } + buffer.append(candidateEnding); + + } + + if (buffer != null && buffer.length() > 0) { + return buffer.toString(); + } + return null; + + } + + /** + * Check for end of line and accumulate a buffer for next time. + * + * @param buffer the current line excluding the candidate ending + * @param candidate a buffer containing accumulated state + * @param next the next character (or -1 for end of file) + * @return true if the values together signify the end of a file + */ + private boolean isEndOfLine(StringBuilder buffer, StringBuilder candidate, int next) { + + if (next == -1) { + return true; + } + + char c = (char) next; + if (ending.charAt(0) == c || candidate.length() > 0) { + candidate.append(c); + } + + if (candidate.length() == 0) { + buffer.append(c); + return false; + } + + boolean end = ending.equals(candidate.toString()); + if (end) { + candidate.delete(0, candidate.length()); + } + else if (candidate.length() >= ending.length()) { + buffer.append(candidate); + candidate.delete(0, candidate.length()); + } + + return end; + + } + } + +} diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/DefaultBufferedReaderFactoryTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/DefaultBufferedReaderFactoryTests.java new file mode 100644 index 000000000..c24942147 --- /dev/null +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/DefaultBufferedReaderFactoryTests.java @@ -0,0 +1,38 @@ +/* + * Copyright 2006-2007 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.item.file; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedReader; + +import org.junit.Test; +import org.springframework.core.io.ByteArrayResource; + +/** + * @author Dave Syer + * + */ +public class DefaultBufferedReaderFactoryTests { + + @Test + public void testCreate() throws Exception { + DefaultBufferedReaderFactory factory = new DefaultBufferedReaderFactory(); + BufferedReader reader = factory.create(new ByteArrayResource("a\nb\nc".getBytes()), "UTF-8"); + assertEquals("a", reader.readLine()); + } + +} diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/SimpleBinaryBufferedReaderFactoryTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/SimpleBinaryBufferedReaderFactoryTests.java new file mode 100644 index 000000000..e4daecc05 --- /dev/null +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/SimpleBinaryBufferedReaderFactoryTests.java @@ -0,0 +1,92 @@ +/* + * Copyright 2006-2007 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.item.file; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedReader; + +import org.junit.Test; +import org.springframework.core.io.ByteArrayResource; + +/** + * @author Dave Syer + * + */ +public class SimpleBinaryBufferedReaderFactoryTests { + + @Test + public void testCreate() throws Exception { + SimpleBinaryBufferedReaderFactory factory = new SimpleBinaryBufferedReaderFactory(); + BufferedReader reader = factory.create(new ByteArrayResource("a\nb".getBytes()), "UTF-8"); + assertEquals("a", reader.readLine()); + assertEquals("b", reader.readLine()); + assertEquals(null, reader.readLine()); + } + + @Test + public void testCreateWithLineEnding() throws Exception { + SimpleBinaryBufferedReaderFactory factory = new SimpleBinaryBufferedReaderFactory(); + factory.setLineEnding("||"); + BufferedReader reader = factory.create(new ByteArrayResource("a||b".getBytes()), "UTF-8"); + assertEquals("a", reader.readLine()); + assertEquals("b", reader.readLine()); + assertEquals(null, reader.readLine()); + } + + @Test + public void testMarkResetWithLineEnding() throws Exception { + SimpleBinaryBufferedReaderFactory factory = new SimpleBinaryBufferedReaderFactory(); + factory.setLineEnding("||"); + BufferedReader reader = factory.create(new ByteArrayResource("a||b||c".getBytes()), "UTF-8"); + assertEquals("a", reader.readLine()); + reader.mark(1024); + assertEquals("b", reader.readLine()); + reader.reset(); + assertEquals("b", reader.readLine()); + assertEquals("c", reader.readLine()); + assertEquals(null, reader.readLine()); + } + + @Test + public void testCreateWithLineEndingAtEnd() throws Exception { + SimpleBinaryBufferedReaderFactory factory = new SimpleBinaryBufferedReaderFactory(); + factory.setLineEnding("||"); + BufferedReader reader = factory.create(new ByteArrayResource("a||".getBytes()), "UTF-8"); + assertEquals("a", reader.readLine()); + assertEquals(null, reader.readLine()); + } + + @Test + public void testCreateWithFalseLineEnding() throws Exception { + SimpleBinaryBufferedReaderFactory factory = new SimpleBinaryBufferedReaderFactory(); + factory.setLineEnding("||"); + BufferedReader reader = factory.create(new ByteArrayResource("a|b||".getBytes()), "UTF-8"); + assertEquals("a|b", reader.readLine()); + assertEquals(null, reader.readLine()); + } + + @Test + public void testCreateWithIncompleteLineEnding() throws Exception { + SimpleBinaryBufferedReaderFactory factory = new SimpleBinaryBufferedReaderFactory(); + factory.setLineEnding("||"); + BufferedReader reader = factory.create(new ByteArrayResource("a||b|".getBytes()), "UTF-8"); + assertEquals("a", reader.readLine()); + assertEquals("b|", reader.readLine()); + assertEquals(null, reader.readLine()); + } + +}