Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

IN PROGRESS - BATCH-863: introduce LineMapper interface to encapsulate string-to-item mapping

Browse Source 
removed FFIR
This commit is contained in:
robokaso
2008-10-07 09:04:31 +00:00
parent 10352b8f56
commit c9efdb0493
16 changed files with 124 additions and 1044 deletions
Download Patch File Download Diff File Expand all files Collapse all files

336
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/FlatFileItemReader.java
View File

@@ -1,336 +0,0 @@
/*
* Copyright 2006-2007 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.item.file;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemStreamException;
import org.springframework.batch.item.ReaderNotOpenException;
import org.springframework.batch.item.UnexpectedInputException;
import org.springframework.batch.item.file.mapping.FieldSet;
import org.springframework.batch.item.file.mapping.FieldSetMapper;
import org.springframework.batch.item.file.separator.DefaultRecordSeparatorPolicy;
import org.springframework.batch.item.file.separator.RecordSeparatorPolicy;
import org.springframework.batch.item.file.transform.AbstractLineTokenizer;
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
import org.springframework.batch.item.file.transform.LineTokenizer;
import org.springframework.batch.item.support.AbstractItemCountingItemStreamItemReader;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.core.io.Resource;
import org.springframework.util.Assert;
import org.springframework.util.ClassUtils;
/**
* This class represents a {@link ItemReader}, that reads lines from text file,
* tokenizes them to structured tuples ({@link FieldSet}s) instances and maps
* the {@link FieldSet}s to domain objects. The location of the file is defined
* by the resource property. To separate the structure of the file,
* {@link LineTokenizer} is used to parse data obtained from the file. <br/>
*
* A {@link FlatFileItemReader} is not thread safe because it maintains state in
* instance variables. <br/>
*
* <p>
* This class supports restart, skipping invalid lines and storing statistics.
* It can be configured to setup {@link FieldSet} column names from the file
* header, skip given number of lines at the beginning of the file.
* </p>
*
* The implementation is *not* thread-safe.
*
* @author Waseem Malik
* @author Tomas Slanina
* @author Robert Kasanicky
* @author Dave Syer
*/
public class FlatFileItemReader<T> extends AbstractItemCountingItemStreamItemReader<T> implements
ResourceAwareItemReaderItemStream<T>, InitializingBean {
private static Log log = LogFactory.getLog(FlatFileItemReader.class);
// default encoding for input files
public static final String DEFAULT_CHARSET = Charset.defaultCharset().name();
private String encoding = DEFAULT_CHARSET;
private Resource resource;
private RecordSeparatorPolicy recordSeparatorPolicy = new DefaultRecordSeparatorPolicy();
private String[] comments = new String[] { "#" };
private int linesToSkip = 0;
private boolean firstLineIsHeader = false;
private LineTokenizer tokenizer = new DelimitedLineTokenizer();
private FieldSetMapper<? extends T> fieldSetMapper;
private int lineCount = 0;
private BufferedReader reader;
private LineCallbackHandler headerCallback;
private boolean noInput;
public FlatFileItemReader() {
setName(ClassUtils.getShortName(FlatFileItemReader.class));
}
/**
* headerCallback will be passed the header line before any items are read.
*/
public void setHeaderCallback(LineCallbackHandler headerCallback) {
this.headerCallback = headerCallback;
}
/**
* Setter for resource property. The location of an input stream that can be
* read.
*
* @param resource
*/
public void setResource(Resource resource) {
this.resource = resource;
}
/**
* Public setter for the recordSeparatorPolicy. Used to determine where the
* line endings are and do things like continue over a line ending if inside
* a quoted string.
*
* @param recordSeparatorPolicy the recordSeparatorPolicy to set
*/
public void setRecordSeparatorPolicy(RecordSeparatorPolicy recordSeparatorPolicy) {
this.recordSeparatorPolicy = recordSeparatorPolicy;
}
/**
* Setter for comment prefixes. Can be used to ignore header lines as well
* by using e.g. the first couple of column names as a prefix.
*
* @param comments an array of comment line prefixes.
*/
public void setComments(String[] comments) {
this.comments = new String[comments.length];
System.arraycopy(comments, 0, this.comments, 0, comments.length);
}
/**
* Indicates whether first line is a header. If the tokenizer is an
* {@link AbstractLineTokenizer} and the column names haven't been set
* already then the header will be used to setup column names. Default is
* <code>false</code>.
*/
public void setFirstLineIsHeader(boolean firstLineIsHeader) {
this.firstLineIsHeader = firstLineIsHeader;
}
/**
* @param lineTokenizer tokenizes each line from file into {@link FieldSet}.
*/
public void setLineTokenizer(LineTokenizer lineTokenizer) {
this.tokenizer = lineTokenizer;
}
/**
* Set the FieldSetMapper to be used for each line.
*
* @param fieldSetMapper
*/
public void setFieldSetMapper(FieldSetMapper<? extends T> fieldSetMapper) {
this.fieldSetMapper = fieldSetMapper;
}
/**
* Public setter for the number of lines to skip at the start of a file. Can
* be used if the file contains a header without useful (column name)
* information, and without a comment delimiter at the beginning of the
* lines.
*
* @param linesToSkip the number of lines to skip
*/
public void setLinesToSkip(int linesToSkip) {
this.linesToSkip = linesToSkip;
}
/**
* Setter for the encoding for this input source. Default value is
* {@link #DEFAULT_CHARSET}.
*
* @param encoding a properties object which possibly contains the encoding
* for this input file;
*/
public void setEncoding(String encoding) {
this.encoding = encoding;
}
public void afterPropertiesSet() throws Exception {
Assert.notNull(fieldSetMapper, "FieldSetMapper must not be null.");
}
/**
* Close the {@link BufferedReader} and reset internal state.
*/
protected void doClose() throws Exception {
if (reader == null) {
return;
}
try {
reader.close();
}
catch (IOException e) {
throw new ItemStreamException("Could not close reader", e);
}
finally {
lineCount = 0;
}
}
/**
* Initialize the {@link BufferedReader} and skip
* {@link #setLinesToSkip(int)} number of lines. Setup column names if
* {@link #setFirstLineIsHeader(boolean)} is <code>true</code>.
*/
protected void doOpen() throws Exception {
Assert.notNull(resource, "Input resource must not be null");
noInput = false;
if (!resource.exists()) {
noInput = true;
log.warn("Input resource does not exist");
return;
}
try {
reader = new BufferedReader(new InputStreamReader(resource.getInputStream(), encoding));
}
catch (IOException e) {
throw new ItemStreamException("Could not open resource", e);
}
log.debug("Opening flat file for reading: " + resource);
for (int i = 0; i < linesToSkip; i++) {
readRecord();
}
if (firstLineIsHeader) {
// skip the header
String firstLine = readRecord();
// set names in tokenizer if they haven't been set already
if (tokenizer instanceof AbstractLineTokenizer && !((AbstractLineTokenizer) tokenizer).hasNames()) {
String[] names = tokenizer.process(firstLine).getValues();
((AbstractLineTokenizer) tokenizer).setNames(names);
}
if (headerCallback != null) {
headerCallback.handleLine(firstLine);
}
}
}
/**
* Reads a line from input, tokenizes is it using the
* {@link #setLineTokenizer(LineTokenizer)} and maps to domain object using
* {@link #setFieldSetMapper(FieldSetMapper)}.
*/
protected T doRead() throws Exception {
if (noInput) {
return null;
}
String record = readRecord();
if (record != null) {
try {
FieldSet tokenizedLine = tokenizer.process(record);
return fieldSetMapper.process(tokenizedLine);
}
catch (Exception ex) {
// add current line count to message and re-throw
throw new FlatFileParseException("Parsing error at line: " + lineCount + " in resource="
+ resource.getDescription() + ", input=[" + record + "]", ex, record, lineCount);
}
}
return null;
}
/**
* @return next line (skip comments).
*/
private String readLine() {
if (reader == null) {
throw new ReaderNotOpenException("Reader must be open before it can be read.");
}
String line = null;
try {
line = this.reader.readLine();
if (line == null) {
return null;
}
lineCount++;
while (isComment(line)) {
line = reader.readLine();
if (line == null) {
return null;
}
lineCount++;
}
}
catch (IOException e) {
throw new UnexpectedInputException("Unable to read from resource '" + resource + "' at line " + lineCount,
e);
}
return line;
}
/**
* @return string corresponding to logical record according to
* {@link #setRecordSeparatorPolicy(RecordSeparatorPolicy)} (might span
* multiple lines in file).
*/
private String readRecord() {
String line = readLine();
String record = line;
if (line != null) {
while (line != null && !recordSeparatorPolicy.isEndOfRecord(record)) {
record = recordSeparatorPolicy.preProcess(record) + (line = readLine());
}
}
return recordSeparatorPolicy.postProcess(record);
}
private boolean isComment(String line) {
for (String prefix : comments) {
if (line.startsWith(prefix)) {
return true;
}
}
return false;
}
}

217
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/FlatFileItemReaderAdvancedTests.java
View File

@@ -1,217 +0,0 @@
/*
* Copyright 2006-2007 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.item.file;
import junit.framework.TestCase;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.item.file.mapping.DefaultFieldSet;
import org.springframework.batch.item.file.mapping.FieldSet;
import org.springframework.batch.item.file.mapping.FieldSetMapper;
import org.springframework.batch.item.file.mapping.PassThroughFieldSetMapper;
import org.springframework.batch.item.file.separator.RecordSeparatorPolicy;
import org.springframework.batch.item.file.transform.LineTokenizer;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource;
import org.springframework.util.ClassUtils;
/**
* Tests for {@link FlatFileItemReader} - skip and restart functionality.
*
* @see FlatFileItemReaderBasicTests
*/
public class FlatFileItemReaderAdvancedTests extends TestCase {
// object under test
private FlatFileItemReader<FieldSet> reader = new FlatFileItemReader<FieldSet>();
// common value used for writing to a file
private String TEST_STRING = "FlatFileInputTemplate-TestData";
private ExecutionContext executionContext;
// simple stub instead of a realistic tokenizer
private LineTokenizer tokenizer = new LineTokenizer() {
public FieldSet process(String line) {
return new DefaultFieldSet(new String[] { line });
}
};
private FieldSetMapper<FieldSet> fieldSetMapper = new PassThroughFieldSetMapper();
/**
* Create inputFile, inject mock/stub dependencies for tested object, initialize the tested object
*/
protected void setUp() throws Exception {
reader.setResource(getInputResource(TEST_STRING));
reader.setLineTokenizer(tokenizer);
reader.setFieldSetMapper(fieldSetMapper);
reader.setSaveState(true);
// context argument is necessary only for the FileLocator, which
// is mocked
executionContext = new ExecutionContext();
}
/**
* Release resources and delete the temporary file
*/
protected void tearDown() throws Exception {
reader.close(null);
}
private Resource getInputResource(String input) {
return new ByteArrayResource(input.getBytes());
}
public void testRestart() throws Exception {
reader.close(null);
reader.setResource(getInputResource("testLine1\ntestLine2\ntestLine3\ntestLine4\ntestLine5\ntestLine6"));
reader.open(executionContext);
// read some records
reader.read();
reader.read();
reader.read();
reader.read();
// get restart data
reader.update(executionContext);
assertEquals(4, executionContext.getLong(ClassUtils.getShortName(FlatFileItemReader.class)
+ ".read.count"));
// close input
reader.close(executionContext);
reader.setResource(getInputResource("testLine1\ntestLine2\ntestLine3\ntestLine4\ntestLine5\ntestLine6"));
// init for restart
reader.open(executionContext);
// read remaining records
assertEquals("[testLine5]", reader.read().toString());
assertEquals("[testLine6]", reader.read().toString());
reader.update(executionContext);
assertEquals(6, executionContext.getLong(ClassUtils.getShortName(FlatFileItemReader.class)
+ ".read.count"));
}
public void testRestartWithCustomRecordSeparatorPolicy() throws Exception {
reader.setResource(getInputResource("testLine1\ntestLine2\ntestLine3\ntestLine4\ntestLine5\ntestLine6"));
reader.setRecordSeparatorPolicy(new RecordSeparatorPolicy() {
// 1 record = 2 lines
boolean pair = true;
public boolean isEndOfRecord(String line) {
pair = !pair;
return pair;
}
public String postProcess(String record) {
return record;
}
public String preProcess(String record) {
return record;
}
});
reader.open(executionContext);
assertEquals("[testLine1testLine2]", reader.read().toString());
assertEquals("[testLine3testLine4]", reader.read().toString());
reader.update(executionContext);
reader.close(executionContext);
reader.open(executionContext);
assertEquals("[testLine5testLine6]", reader.read().toString());
}
public void testRestartWithHeader() throws Exception {
reader.close(null);
reader.setResource(getInputResource("header\ntestLine1\ntestLine2\ntestLine3\ntestLine4\ntestLine5\ntestLine6"));
reader.setFirstLineIsHeader(true);
reader.open(executionContext);
// read some records
reader.read();
reader.read();
// get restart data
reader.update(executionContext);
// read next two records
reader.read();
reader.read();
assertEquals(2, executionContext.getLong(ClassUtils.getShortName(FlatFileItemReader.class)
+ ".read.count"));
// close input
reader.close(executionContext);
reader.setResource(getInputResource("header\ntestLine1\ntestLine2\ntestLine3\ntestLine4\ntestLine5\ntestLine6"));
// init for restart
reader.open(executionContext);
// read remaining records
assertEquals("[testLine3]", reader.read().toString());
assertEquals("[testLine4]", reader.read().toString());
reader.update(executionContext);
assertEquals(4, executionContext.getLong(ClassUtils.getShortName(FlatFileItemReader.class)
+ ".read.count"));
}
public void testRestartWithSkippedLines() throws Exception {
reader.close(null);
reader.setResource(getInputResource("header\nignoreme\n\ntestLine1\ntestLine2\ntestLine3\ntestLine4\ntestLine5\ntestLine6"));
reader.setLinesToSkip(2);
reader.open(executionContext);
// read some records
reader.read();
reader.read();
// get restart data
reader.update(executionContext);
// read next two records
reader.read();
reader.read();
assertEquals(2, executionContext.getLong(ClassUtils.getShortName(FlatFileItemReader.class)
+ ".read.count"));
// close input
reader.close(executionContext);
reader.setResource(getInputResource("header\nignoreme\ntestLine1\ntestLine2\ntestLine3\ntestLine4\ntestLine5\ntestLine6"));
// init for restart
reader.open(executionContext);
// read remaining records
assertEquals("[testLine3]", reader.read().toString());
assertEquals("[testLine4]", reader.read().toString());
reader.update(executionContext);
assertEquals(4, executionContext.getLong(ClassUtils.getShortName(FlatFileItemReader.class)
+ ".read.count"));
}
}

370
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/FlatFileItemReaderBasicTests.java
View File

@@ -1,370 +0,0 @@
/*
* Copyright 2006-2007 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.item.file;
import static org.easymock.EasyMock.createStrictMock;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.io.InputStream;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.item.ItemStreamException;
import org.springframework.batch.item.ReaderNotOpenException;
import org.springframework.batch.item.file.mapping.DefaultFieldSet;
import org.springframework.batch.item.file.mapping.FieldSet;
import org.springframework.batch.item.file.mapping.FieldSetMapper;
import org.springframework.batch.item.file.mapping.PassThroughFieldSetMapper;
import org.springframework.batch.item.file.separator.DefaultRecordSeparatorPolicy;
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
import org.springframework.batch.item.file.transform.LineTokenizer;
import org.springframework.core.io.AbstractResource;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource;
/**
* Tests for {@link FlatFileItemReader} - the fundamental item reading functionality.
*
* @see FlatFileItemReaderAdvancedTests
* @author Dave Syer
*/
public class FlatFileItemReaderBasicTests {
// object under test
private FlatFileItemReader<FieldSet> itemReader = new FlatFileItemReader<FieldSet>();
// common value used for writing to a file
private String TEST_STRING = "FlatFileInputTemplate-TestData";
private String TEST_OUTPUT = "[FlatFileInputTemplate-TestData]";
private ExecutionContext executionContext;
// simple stub instead of a realistic tokenizer
private LineTokenizer tokenizer = new LineTokenizer() {
public FieldSet process(String line) {
return new DefaultFieldSet(new String[] { line });
}
};
private FieldSetMapper<FieldSet> fieldSetMapper = new PassThroughFieldSetMapper();
/**
* Create inputFile, inject mock/stub dependencies for tested object, initialize the tested object
*/
@Before
public void setUp() throws Exception {
itemReader.setResource(getInputResource(TEST_STRING));
itemReader.setLineTokenizer(tokenizer);
itemReader.setFieldSetMapper(fieldSetMapper);
itemReader.afterPropertiesSet();
executionContext = new ExecutionContext();
}
/**
* Release resources.
*/
@After
public void tearDown() throws Exception {
itemReader.close(null);
}
private Resource getInputResource(String input) {
return new ByteArrayResource(input.getBytes());
}
/**
* Regular usage of <code>read</code> method
*/
@Test
public void testRead() throws Exception {
itemReader.open(executionContext);
assertEquals("[FlatFileInputTemplate-TestData]", itemReader.read().toString());
}
/**
* Regular usage of <code>read</code> method
*/
@Test
public void testReadExhausted() throws Exception {
itemReader.open(executionContext);
assertEquals("[FlatFileInputTemplate-TestData]", itemReader.read().toString());
assertEquals(null, itemReader.read());
}
/**
* Regular usage of <code>read</code> method
*/
@Test
public void testReadWithTokenizerError() throws Exception {
itemReader.setLineTokenizer(new LineTokenizer() {
public FieldSet process(String line) {
throw new RuntimeException("foo");
}
});
try {
itemReader.open(executionContext);
itemReader.read();
fail("Expected ParsingException");
} catch (FlatFileParseException e) {
assertEquals(e.getInput(), TEST_STRING);
assertEquals(e.getLineNumber(), 1);
}
}
@Test
public void testReadWithMapperError() throws Exception {
itemReader.setFieldSetMapper(new FieldSetMapper<FieldSet>() {
public FieldSet process(FieldSet fs) {
throw new RuntimeException("foo");
}
});
try {
itemReader.open(executionContext);
itemReader.read();
fail("Expected ParsingException");
} catch (FlatFileParseException e) {
assertEquals(e.getInput(), TEST_STRING);
assertEquals(e.getLineNumber(), 1);
}
}
@Test
public void testReadBeforeOpen() throws Exception {
itemReader = new FlatFileItemReader<FieldSet>();
itemReader.setResource(getInputResource(TEST_STRING));
itemReader.setFieldSetMapper(fieldSetMapper);
try {
itemReader.read();
fail("Expected ReaderNotOpenException");
} catch (ReaderNotOpenException e) {
assertTrue(contains(e.getMessage(), "open"));
}
}
@Test
public void testCloseBeforeOpen() throws Exception {
itemReader = new FlatFileItemReader<FieldSet>();
itemReader.setResource(getInputResource(TEST_STRING));
itemReader.setFieldSetMapper(fieldSetMapper);
itemReader.close(null);
// The open does not happen automatically on a read...
itemReader.open(executionContext);
assertEquals("[FlatFileInputTemplate-TestData]", itemReader.read().toString());
}
@Test
public void testInitializationWithNullResource() throws Exception {
itemReader = new FlatFileItemReader<FieldSet>();
try {
itemReader.afterPropertiesSet();
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
// expected
}
}
@Test
public void testOpenTwiceHasNoEffect() throws Exception {
itemReader.open(executionContext);
testRead();
}
@Test
public void testSetValidEncoding() throws Exception {
itemReader = new FlatFileItemReader<FieldSet>();
itemReader.setEncoding("UTF-8");
itemReader.setResource(getInputResource(TEST_STRING));
itemReader.setFieldSetMapper(fieldSetMapper);
itemReader.open(executionContext);
testRead();
}
@Test
public void testSetNullEncoding() throws Exception {
itemReader = new FlatFileItemReader<FieldSet>();
itemReader.setEncoding(null);
itemReader.setResource(getInputResource(TEST_STRING));
try {
itemReader.open(executionContext);
fail("Expected IllegalArgumentException");
} catch (ItemStreamException e) {
// expected
}
}
@Test
public void testSetInvalidEncoding() throws Exception {
itemReader = new FlatFileItemReader<FieldSet>();
itemReader.setEncoding("foo");
itemReader.setResource(getInputResource(TEST_STRING));
try {
itemReader.open(executionContext);
fail("Expected BatchEnvironmentException");
} catch (ItemStreamException e) {
// expected
assertEquals("Failed to initialize the reader", e.getMessage());
assertEquals("foo", e.getCause().getCause().getMessage());
}
}
@Test
public void testEncoding() throws Exception {
itemReader.setEncoding("UTF-8");
testRead();
}
@Test
public void testRecordSeparator() throws Exception {
itemReader.setRecordSeparatorPolicy(new DefaultRecordSeparatorPolicy());
testRead();
}
@Test
public void testComments() throws Exception {
itemReader.setResource(getInputResource("% Comment\n" + TEST_STRING));
itemReader.setComments(new String[] { "%" });
testRead();
}
/**
* Header line is skipped and used to setup fieldSet column names.
* It is also passed to header callback.
*/
@Test
public void testColumnNamesInHeader() throws Exception {
final String INPUT = "name1|name2\nvalue1|value2\nvalue3|value4";
LineCallbackHandler headerCallback = createStrictMock(LineCallbackHandler.class);
headerCallback.handleLine("name1|name2");
replay(headerCallback);
itemReader = new FlatFileItemReader<FieldSet>();
itemReader.setResource(getInputResource(INPUT));
itemReader.setLineTokenizer(new DelimitedLineTokenizer('|'));
itemReader.setFieldSetMapper(fieldSetMapper);
itemReader.setFirstLineIsHeader(true);
itemReader.setHeaderCallback(headerCallback);
itemReader.afterPropertiesSet();
itemReader.open(executionContext);
FieldSet fs = itemReader.read();
assertEquals("value1", fs.readString("name1"));
assertEquals("value2", fs.readString("name2"));
fs = itemReader.read();
assertEquals("value3", fs.readString("name1"));
assertEquals("value4", fs.readString("name2"));
verify(headerCallback);
}
/**
* Header line is skipped and used to setup fieldSet column names.
*/
@Test
public void testLinesToSkip() throws Exception {
final String INPUT = "foo bar spam\none two\nthree four";
itemReader = new FlatFileItemReader<FieldSet>();
itemReader.setResource(getInputResource(INPUT));
itemReader.setLineTokenizer(new DelimitedLineTokenizer(' '));
itemReader.setFieldSetMapper(fieldSetMapper);
itemReader.setLinesToSkip(1);
itemReader.afterPropertiesSet();
itemReader.open(executionContext);
FieldSet fs = itemReader.read();
assertEquals("one", fs.readString(0));
assertEquals("two", fs.readString(1));
fs = itemReader.read();
assertEquals("three", fs.readString(0));
assertEquals("four", fs.readString(1));
}
@Test
public void testNonExistantResource() throws Exception {
Resource resource = new NonExistentResource();
FlatFileItemReader<FieldSet> testReader = new FlatFileItemReader<FieldSet>();
testReader.setResource(resource);
testReader.setLineTokenizer(tokenizer);
testReader.setFieldSetMapper(fieldSetMapper);
testReader.setResource(resource);
// afterPropertiesSet should only throw an exception if the Resource is null
testReader.afterPropertiesSet();
testReader.open(executionContext);
assertNull(testReader.read());
}
@Test
public void testRuntimeFileCreation() throws Exception {
Resource resource = new NonExistentResource();
FlatFileItemReader<FieldSet> testReader = new FlatFileItemReader<FieldSet>();
testReader.setResource(resource);
testReader.setLineTokenizer(tokenizer);
testReader.setFieldSetMapper(fieldSetMapper);
testReader.setResource(resource);
// afterPropertiesSet should only throw an exception if the Resource is null
testReader.afterPropertiesSet();
// replace the resource to simulate runtime resource creation
testReader.setResource(getInputResource(TEST_STRING));
testReader.open(executionContext);
assertEquals(TEST_OUTPUT, testReader.read().toString());
}
private boolean contains(String str, String searchStr) {
return str.indexOf(searchStr) != -1;
}
private static class NonExistentResource extends AbstractResource {
public NonExistentResource() {
}
public boolean exists() {
return false;
}
public String getDescription() {
return "NonExistantResource";
}
public InputStream getInputStream() throws IOException {
return null;
}
}
}

46
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/FlatFileItemReaderCommonTests.java
View File

@@ -1,46 +0,0 @@
package org.springframework.batch.item.file;
import org.springframework.batch.item.CommonItemStreamItemReaderTests;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.file.mapping.FieldSet;
import org.springframework.batch.item.file.mapping.FieldSetMapper;
import org.springframework.batch.item.sample.Foo;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource;
import org.junit.runner.RunWith;
import org.junit.internal.runners.JUnit4ClassRunner;
@RunWith(JUnit4ClassRunner.class)
public class FlatFileItemReaderCommonTests extends CommonItemStreamItemReaderTests {
private static final String FOOS = "1 \n 2 \n 3 \n 4 \n 5 \n";
protected ItemReader<Foo> getItemReader() throws Exception {
FlatFileItemReader<Foo> tested = new FlatFileItemReader<Foo>();
Resource resource = new ByteArrayResource(FOOS.getBytes());
tested.setResource(resource);
tested.setFieldSetMapper(new FieldSetMapper<Foo>() {
public Foo process(FieldSet fs) {
Foo foo = new Foo();
foo.setValue(fs.readInt(0));
return foo;
}
});
tested.setSaveState(true);
tested.afterPropertiesSet();
return tested;
}
protected void pointToEmptyInput(ItemReader<Foo> tested) throws Exception {
FlatFileItemReader<Foo> reader = (FlatFileItemReader<Foo>) tested;
reader.close(new ExecutionContext());
reader.setResource(new ByteArrayResource("".getBytes()));
reader.afterPropertiesSet();
reader.open(new ExecutionContext());
}
}

5
spring-batch-samples/src/main/java/org/springframework/batch/sample/support/HeaderCopyCallback.java
View File

@@ -3,15 +3,14 @@ package org.springframework.batch.sample.support;
import java.io.IOException;
import java.io.Writer;
import org.springframework.batch.item.file.FileWriterCallback;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.FlatFileItemWriter;
import org.springframework.batch.item.file.LineCallbackHandler;
import org.springframework.batch.item.file.ResourceLineReader;
import org.springframework.util.Assert;
/**
* Designed to be registered with both {@link FlatFileItemReader} and
* Designed to be registered with both {@link ResourceLineReader} and
* {@link FlatFileItemWriter} and copy header line from input file to output
* file.
*/

23
spring-batch-samples/src/main/resources/jobs/beanWrapperMapperSampleJob.xml
View File

@@ -35,20 +35,27 @@
</bean>
<!-- INFRASTRUCTURE SETUP -->
<bean id="tradeFileItemReader" class="org.springframework.batch.item.file.FlatFileItemReader">
<bean id="tradeFileItemReader" class="org.springframework.batch.item.file.ResourceLineReader">
<property name="resource"
value="classpath:data/beanWrapperMapperSampleJob/input/20070122.teststream.ImportTradeDataStep.txt" />
<property name="lineTokenizer" ref="tradeTokenizer" />
<property name="fieldSetMapper" ref="tradeFieldSetMapper" />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer" ref="tradeTokenizer" />
<property name="fieldSetMapper" ref="tradeFieldSetMapper" />
</bean>
</property>
</bean>
<bean id="personFileItemReader" class="org.springframework.batch.item.file.FlatFileItemReader">
<bean id="personFileItemReader" class="org.springframework.batch.item.file.ResourceLineReader">
<property name="resource"
value="classpath:data/beanWrapperMapperSampleJob/input/20070122.teststream.ImportPersonDataStep.txt" />
<property name="lineTokenizer" ref="personTokenizer" />
<property name="fieldSetMapper" ref="personFieldSetMapper" />
<!-- <property name="validator" ref="fixedValidator" />-->
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer" ref="personTokenizer" />
<property name="fieldSetMapper" ref="personFieldSetMapper" />
</bean>
</property>
</bean>
<bean id="tradeTokenizer" class="org.springframework.batch.item.file.transform.FixedLengthTokenizer">

10
spring-batch-samples/src/main/resources/jobs/compositeItemWriterSampleJob.xml
View File

@@ -45,11 +45,15 @@
</bean>
<bean id="fileItemReader"
class="org.springframework.batch.item.file.FlatFileItemReader">
class="org.springframework.batch.item.file.ResourceLineReader">
<property name="resource"
value="classpath:data/fixedLengthImportJob/input/20070122.teststream.ImportTradeDataStep.txt" />
<property name="lineTokenizer" ref="fixedFileTokenizer" />
<property name="fieldSetMapper" ref="fieldSetMapper" />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer" ref="fixedFileTokenizer" />
<property name="fieldSetMapper" ref="fieldSetMapper" />
</bean>
</property>
</bean>
<bean id="fixedFileTokenizer"

48
spring-batch-samples/src/main/resources/jobs/footballJob.xml
View File

@@ -59,37 +59,45 @@
</bean>
<bean id="playerFileItemReader"
class="org.springframework.batch.item.file.FlatFileItemReader">
class="org.springframework.batch.item.file.ResourceLineReader">
<property name="resource"
value="classpath:data/footballjob/input/${player.file.name}" />
<property name="lineTokenizer">
<bean
class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
<property name="names"
value="ID,lastName,firstName,position,birthYear,debutYear" />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer">
<bean
class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
<property name="names"
value="ID,lastName,firstName,position,birthYear,debutYear" />
</bean>
</property>
<property name="fieldSetMapper">
<bean
class="org.springframework.batch.sample.domain.football.internal.PlayerFieldSetMapper" />
</property>
</bean>
</property>
<property name="fieldSetMapper">
<bean
class="org.springframework.batch.sample.domain.football.internal.PlayerFieldSetMapper" />
</property>
</bean>
<bean id="gameFileItemReader"
class="org.springframework.batch.item.file.FlatFileItemReader">
class="org.springframework.batch.item.file.ResourceLineReader">
<property name="resource"
value="classpath:data/footballjob/input/${games.file.name}" />
<property name="lineTokenizer">
<bean
class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
<property name="names"
value="id,year,team,week,opponent,completes,attempts,passingYards,passingTd,interceptions,rushes,rushYards,receptions,receptionYards,totalTd" />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer">
<bean
class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
<property name="names"
value="id,year,team,week,opponent,completes,attempts,passingYards,passingTd,interceptions,rushes,rushYards,receptions,receptionYards,totalTd" />
</bean>
</property>
<property name="fieldSetMapper">
<bean
class="org.springframework.batch.sample.domain.football.internal.GameFieldSetMapper" />
</property>
</bean>
</property>
<property name="fieldSetMapper">
<bean
class="org.springframework.batch.sample.domain.football.internal.GameFieldSetMapper" />
</property>
</bean>
<bean id="playerSummarizationSource"

26
spring-batch-samples/src/main/resources/jobs/headerFooterSample.xml
View File

@@ -28,20 +28,24 @@
</bean>
</property>
</bean>
<bean id="reader" class="org.springframework.batch.item.file.FlatFileItemReader">
<bean id="reader" class="org.springframework.batch.item.file.ResourceLineReader">
<property name="resource" ref="inputResource" />
<property name="lineTokenizer">
<bean
class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
<property name="delimiter" value="," />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer">
<bean
class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
<property name="delimiter" value="," />
</bean>
</property>
<property name="fieldSetMapper">
<bean
class="org.springframework.batch.item.file.mapping.PassThroughFieldSetMapper" />
</property>
</bean>
</property>
<property name="fieldSetMapper">
<bean
class="org.springframework.batch.item.file.mapping.PassThroughFieldSetMapper" />
</property>
<property name="headerCallback" ref="headerCopier" />
<property name="firstLineIsHeader" value="true" />
<property name="skippedLinesCallback" ref="headerCopier" />
<property name="linesToSkip" value="1" />
</bean>
<bean id="writer" class="org.springframework.batch.item.file.FlatFileItemWriter">
<property name="resource" ref="outputResource" />

10
spring-batch-samples/src/main/resources/jobs/multiResourceJob.xml
View File

@@ -42,11 +42,15 @@
autowire-candidate="false" />
<bean id="flatFileItemReader"
class="org.springframework.batch.item.file.FlatFileItemReader">
class="org.springframework.batch.item.file.ResourceLineReader">
<property name="resource"
value="classpath:data/fixedLengthImportJob/input/20070122.teststream.ImportTradeDataStep.txt" />
<property name="lineTokenizer" ref="fixedFileTokenizer" />
<property name="fieldSetMapper" ref="fieldSetMapper" />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer" ref="fixedFileTokenizer" />
<property name="fieldSetMapper" ref="fieldSetMapper" />
</bean>
</property>
</bean>
<bean id="fixedFileTokenizer"

15
spring-batch-samples/src/main/resources/jobs/multilineJob.xml
View File

@@ -33,16 +33,19 @@
</bean>
<bean id="fileItemReader"
class="org.springframework.batch.item.file.FlatFileItemReader">
class="org.springframework.batch.item.file.ResourceLineReader">
<property name="resource"
value="classpath:data/multilineJob/input/20070122.teststream.multilineStep.txt" />
<property name="lineTokenizer" ref="fixedFileDescriptor" />
<property name="fieldSetMapper">
<bean class="org.springframework.batch.item.support.AggregateItemFieldSetMapper">
<property name="delegate" ref="tradeLineMapper" />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer" ref="fixedFileDescriptor" />
<property name="fieldSetMapper">
<bean class="org.springframework.batch.item.support.AggregateItemFieldSetMapper">
<property name="delegate" ref="tradeLineMapper" />
</bean>
</property>
</bean>
</property>
<!-- <property name="validator" ref="fixedValidator" /> -->
</bean>
<bean id="tradeLineMapper"

14
spring-batch-samples/src/main/resources/jobs/multilineOrderIo.xml
View File

@@ -6,11 +6,15 @@
http://www.springframework.org/schema/aop
http://www.springframework.org/schema/aop/spring-aop-2.0.xsd">
<bean id="fileItemReader" class="org.springframework.batch.item.file.FlatFileItemReader">
<property name="resource" ref="fileInputLocator" />
<property name="lineTokenizer" ref="orderFileTokenizer" />
<property name="fieldSetMapper">
<bean class="org.springframework.batch.item.file.mapping.PassThroughFieldSetMapper" />
<bean id="fileItemReader" class="org.springframework.batch.item.file.ResourceLineReader">
<property name="resource" ref="fileInputLocator" />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer" ref="orderFileTokenizer" />
<property name="fieldSetMapper">
<bean class="org.springframework.batch.item.file.mapping.PassThroughFieldSetMapper" />
</property>
</bean>
</property>
</bean>

10
spring-batch-samples/src/main/resources/jobs/parallelJob.xml
View File

@@ -66,11 +66,15 @@
<!-- This input source is injected into the test case to verify the output - not used by the job at all -->
<bean id="testItemReader"
class="org.springframework.batch.item.file.FlatFileItemReader">
class="org.springframework.batch.item.file.ResourceLineReader">
<property name="resource"
value="classpath:data/fixedLengthImportJob/input/20070122.teststream.ImportTradeDataStep.txt" />
<property name="lineTokenizer" ref="fixedFileTokenizer" />
<property name="fieldSetMapper" ref="fieldSetMapper" />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer" ref="fixedFileTokenizer" />
<property name="fieldSetMapper" ref="fieldSetMapper" />
</bean>
</property>
</bean>
<bean id="fileItemReader" parent="testItemReader"

10
spring-batch-samples/src/main/resources/jobs/restartSample.xml
View File

@@ -38,11 +38,15 @@
</bean>
<bean id="fileItemReader"
class="org.springframework.batch.item.file.FlatFileItemReader">
class="org.springframework.batch.item.file.ResourceLineReader">
<property name="resource"
value="classpath:data/fixedLengthImportJob/input/20070122.teststream.ImportTradeDataStep.txt" />
<property name="lineTokenizer" ref="fixedFileDescriptor" />
<property name="fieldSetMapper" ref="fieldSetMapper" />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer" ref="fixedFileDescriptor" />
<property name="fieldSetMapper" ref="fieldSetMapper" />
</bean>
</property>
<property name="saveState" value="true" />
</bean>

14
spring-batch-samples/src/main/resources/jobs/tradeJobIo.xml
View File

@@ -38,13 +38,17 @@
</property>
</bean>
<bean class="org.springframework.batch.item.file.FlatFileItemReader"
<bean class="org.springframework.batch.item.file.ResourceLineReader"
id="fileItemReader">
<property name="resource" ref="fileLocator" />
<property name="lineTokenizer" ref="tradeFileDescriptor" />
<property name="fieldSetMapper">
<bean
class="org.springframework.batch.sample.domain.trade.internal.TradeFieldSetMapper" />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer" ref="tradeFileDescriptor" />
<property name="fieldSetMapper">
<bean
class="org.springframework.batch.sample.domain.trade.internal.TradeFieldSetMapper" />
</property>
</bean>
</property>
</bean>

14
spring-batch-samples/src/test/java/org/springframework/batch/sample/FixedLengthImportJobFunctionalTests.java
View File

@@ -29,7 +29,8 @@ import javax.sql.DataSource;
import org.junit.Before;
import org.junit.runner.RunWith;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.ResourceLineReader;
import org.springframework.batch.item.file.mapping.DefaultLineMapper;
import org.springframework.batch.item.file.mapping.FieldSetMapper;
import org.springframework.batch.item.file.transform.LineTokenizer;
import org.springframework.batch.sample.domain.trade.Trade;
@@ -52,7 +53,7 @@ public class FixedLengthImportJobFunctionalTests extends AbstractValidatingBatch
//auto-injected attributes
private SimpleJdbcTemplate simpleJdbcTemplate;
private Resource fileLocator;
private FlatFileItemReader<Trade> inputSource;
private ResourceLineReader<Trade> inputSource;
private LineTokenizer lineTokenizer;
@Autowired
@@ -70,12 +71,15 @@ public class FixedLengthImportJobFunctionalTests extends AbstractValidatingBatch
public void onSetUp() throws Exception {
simpleJdbcTemplate.update("delete from TRADE");
fileLocator = new ClassPathResource("data/fixedLengthImportJob/input/20070122.teststream.ImportTradeDataStep.txt");
inputSource = new FlatFileItemReader<Trade>();
inputSource = new ResourceLineReader<Trade>();
FieldSetMapper<Trade> mapper = new TradeFieldSetMapper();
inputSource.setFieldSetMapper(mapper);
DefaultLineMapper<Trade> lineMapper = new DefaultLineMapper<Trade>();
lineMapper.setLineTokenizer(lineTokenizer);
lineMapper.setFieldSetMapper(mapper);
inputSource.setLineMapper(lineMapper);
inputSource.setLineTokenizer(lineTokenizer);
inputSource.setResource(fileLocator);
}