From 63b097a2d269357a3c736ff17e434b90d20b9224 Mon Sep 17 00:00:00 2001 From: dsyer Date: Tue, 21 Aug 2007 11:04:18 +0000 Subject: [PATCH] OPEN - issue BATCH-105: The functionality in "DefaultFlatFileItemProvider" (validating and mapping) should be provided by the infrastructure layer (DefaultFlatFileInputSource). http://opensource.atlassian.com/projects/spring/browse/BATCH-105 --- .../processor/CompositeItemProcessor.java | 143 +++++++++++++ .../processor/OutputSourceItemProcessor.java | 82 ++++++++ .../batch/item/processor/package.html | 7 + .../provider/DefaultFlatFileItemProvider.java | 105 ++++++++++ .../provider/InputSourceItemProvider.java | 97 +++++++++ .../DefaultFlatFileItemProviderTests.java | 189 ++++++++++++++++++ .../InputSourceItemProviderTests.java | 123 ++++++++++++ 7 files changed, 746 insertions(+) create mode 100644 infrastructure/src/main/java/org/springframework/batch/item/processor/CompositeItemProcessor.java create mode 100644 infrastructure/src/main/java/org/springframework/batch/item/processor/OutputSourceItemProcessor.java create mode 100644 infrastructure/src/main/java/org/springframework/batch/item/processor/package.html create mode 100644 infrastructure/src/main/java/org/springframework/batch/item/provider/DefaultFlatFileItemProvider.java create mode 100644 infrastructure/src/main/java/org/springframework/batch/item/provider/InputSourceItemProvider.java create mode 100644 infrastructure/src/test/java/org/springframework/batch/item/provider/DefaultFlatFileItemProviderTests.java create mode 100644 infrastructure/src/test/java/org/springframework/batch/item/provider/InputSourceItemProviderTests.java diff --git a/infrastructure/src/main/java/org/springframework/batch/item/processor/CompositeItemProcessor.java b/infrastructure/src/main/java/org/springframework/batch/item/processor/CompositeItemProcessor.java new file mode 100644 index 000000000..231fa66fb --- /dev/null +++ b/infrastructure/src/main/java/org/springframework/batch/item/processor/CompositeItemProcessor.java @@ -0,0 +1,143 @@ +package org.springframework.batch.item.processor; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.restart.GenericRestartData; +import org.springframework.batch.restart.RestartData; +import org.springframework.batch.restart.Restartable; +import org.springframework.batch.statistics.StatisticsProvider; + +/** + * Runs a collection of ItemProcessors in fixed-order sequence. + * + * @author Robert Kasanicky + */ +public class CompositeItemProcessor implements ItemProcessor, Restartable, StatisticsProvider { + + private static final String SEPARATOR = "#"; + + private List itemProcessors; + + /** + * Calls injected ItemProcessors in order. + */ + public void process(Object data) throws Exception { + for (Iterator iterator = itemProcessors.listIterator(); iterator.hasNext();) { + ((ItemProcessor) iterator.next()).process(data); + } + } + + /** + * Compound restart data of all injected (Restartable) ItemProcessors, property keys are + * prefixed with list index of the ItemProcessor. + */ + public RestartData getRestartData() { + Properties props = createCompoundProperties(new PropertiesExtractor() { + public Properties extractProperties(Object o) { + if (o instanceof Restartable) { + return ((Restartable)o).getRestartData().getProperties(); + } else { + return null; + } + } + }); + return new GenericRestartData(props); + } + + /** + * @param data contains values of restart data, property keys are expected to be prefixed with + * list index of the ItemProcessor. + */ + public void restoreFrom(RestartData data) { + if (data == null || data.getProperties() == null) { + // do nothing + return; + } + + List restartDataList = parseProperties(data.getProperties()); + + // iterators would make the loop below less readable + for (int i=0; i < itemProcessors.size(); i++) { + if (itemProcessors.get(i) instanceof Restartable) { + ((Restartable) itemProcessors.get(i)).restoreFrom((RestartData) restartDataList.get(i)); + } + } + + } + + /** + * @return Properties containing statistics of all injected ItemProcessors, + * property keys are prefixed with the list index of the ItemProcessor. + */ + public Properties getStatistics() { + return createCompoundProperties(new PropertiesExtractor() { + public Properties extractProperties(Object o) { + if (o instanceof StatisticsProvider){ + return ((StatisticsProvider) o).getStatistics(); + } else { + return null; + } + } + }); + } + + public void setItemProcessors(List itemProcessors) { + this.itemProcessors = itemProcessors; + } + + /** + * Parses compound properties into a list of RestartData. + */ + private List parseProperties(Properties props) { + List restartDataList = new ArrayList(itemProcessors.size()); + for (int i = 0; i + +

+Specific implementations of item processing concerns. +

+ + diff --git a/infrastructure/src/main/java/org/springframework/batch/item/provider/DefaultFlatFileItemProvider.java b/infrastructure/src/main/java/org/springframework/batch/item/provider/DefaultFlatFileItemProvider.java new file mode 100644 index 000000000..853a51582 --- /dev/null +++ b/infrastructure/src/main/java/org/springframework/batch/item/provider/DefaultFlatFileItemProvider.java @@ -0,0 +1,105 @@ +/* + * Copyright 2006-2007 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.item.provider; + +import java.util.Properties; + +import org.springframework.batch.io.file.FieldSet; +import org.springframework.batch.io.file.FieldSetMapper; +import org.springframework.batch.item.validator.Validator; +import org.springframework.batch.restart.RestartData; +import org.springframework.batch.restart.Restartable; +import org.springframework.batch.statistics.StatisticsProvider; + +/** + * + * Uses a {@link FieldSetMapper} to convert each line from an input source. Also + * adds {@link Restartable} as mandatory behaviour, delegating to the parent + * provider's input source. + * + * @author Dave Syer + */ +public class DefaultFlatFileItemProvider extends AbstractFieldSetItemProvider implements Restartable, + StatisticsProvider { + + private FieldSetMapper mapper; + private Validator validator; + + /* + * (non-Javadoc) + * @see org.springframework.batch.item.provider.AbstractFieldSetItemProvider#doNext(org.springframework.batch.io.line.FieldSet) + */ + protected Object transform(FieldSet fieldSet) { + Object value = mapper.mapLine(fieldSet); + if (validator!=null) { + validator.validate(value); + } + return value; + } + + /** + * @param mapper the mapper to set + */ + public void setMapper(FieldSetMapper mapper) { + this.mapper = mapper; + } + + /** + * @param validator the validator to set + */ + public void setValidator(Validator validator) { + this.validator = validator; + } + + /** + * @see Restartable#getRestartData() + * @throws IllegalStateException if the parent template is not itself + * {@link Restartable}. + */ + public RestartData getRestartData() { + if (!(source instanceof Restartable)) { + throw new IllegalStateException("Input Template is not Restartable"); + } + return ((Restartable) source).getRestartData(); + } + + /** + * @see Restartable#restoreFrom(RestartData) + * @throws IllegalStateException if the parent template is not itself + * {@link Restartable}. + */ + public void restoreFrom(RestartData data) { + if (!(source instanceof Restartable)) { + throw new IllegalStateException("Input Template is not Restartable"); + } + ((Restartable) source).restoreFrom(data); + } + + /** + * @return delegates to the parent template of it is a + * {@link StatisticsProvider}, otherwise returns an empty + * {@link Properties} instance. + * @see StatisticsProvider#getStatistics() + */ + public Properties getStatistics() { + if (!(source instanceof StatisticsProvider)) { + return new Properties(); + } + return ((StatisticsProvider) source).getStatistics(); + } + +} diff --git a/infrastructure/src/main/java/org/springframework/batch/item/provider/InputSourceItemProvider.java b/infrastructure/src/main/java/org/springframework/batch/item/provider/InputSourceItemProvider.java new file mode 100644 index 000000000..9eec73921 --- /dev/null +++ b/infrastructure/src/main/java/org/springframework/batch/item/provider/InputSourceItemProvider.java @@ -0,0 +1,97 @@ +/* + * Copyright 2006-2007 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.item.provider; + +import java.util.Properties; + +import org.springframework.batch.io.InputSource; +import org.springframework.batch.io.Skippable; +import org.springframework.batch.restart.RestartData; +import org.springframework.batch.restart.Restartable; +import org.springframework.batch.statistics.StatisticsProvider; + +/** + * Simple wrapper around {@link InputSource}. The input source is expected to + * take care of open and close operations. If necessary it should be registered + * as a step scoped bean to ensure that the lifecycle methods are called. + * + * @author Dave Syer + */ +public class InputSourceItemProvider extends AbstractItemProvider implements Restartable, StatisticsProvider, Skippable { + + private InputSource source; + + /** + * Get the next object from the input source. + * @see org.springframework.batch.item.ItemProvider#next() + */ + public Object next() { + Object value = source.read(); + return value; + } + + /** + * @see Restartable#getRestartData() + * @throws IllegalStateException if the parent template is not itself + * {@link Restartable}. + */ + public RestartData getRestartData() { + if (!(source instanceof Restartable)) { + throw new IllegalStateException("Input Template is not Restartable"); + } + return ((Restartable) source).getRestartData(); + } + + /** + * @see Restartable#restoreFrom(RestartData) + * @throws IllegalStateException if the parent template is not itself + * {@link Restartable}. + */ + public void restoreFrom(RestartData data) { + if (!(source instanceof Restartable)) { + throw new IllegalStateException("Input Template is not Restartable"); + } + ((Restartable) source).restoreFrom(data); + } + + /** + * @return delegates to the parent template of it is a + * {@link StatisticsProvider}, otherwise returns an empty + * {@link Properties} instance. + * @see StatisticsProvider#getStatistics() + */ + public Properties getStatistics() { + if (!(source instanceof StatisticsProvider)) { + return new Properties(); + } + return ((StatisticsProvider) source).getStatistics(); + } + + /** + * Setter for input source. + * @param source + */ + public void setInputSource(InputSource source) { + this.source = source; + } + + public void skip() { + if (source instanceof Skippable) { + ((Skippable)source).skip(); + } + } +} diff --git a/infrastructure/src/test/java/org/springframework/batch/item/provider/DefaultFlatFileItemProviderTests.java b/infrastructure/src/test/java/org/springframework/batch/item/provider/DefaultFlatFileItemProviderTests.java new file mode 100644 index 000000000..e2666aadc --- /dev/null +++ b/infrastructure/src/test/java/org/springframework/batch/item/provider/DefaultFlatFileItemProviderTests.java @@ -0,0 +1,189 @@ +/* + * Copyright 2006-2007 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.item.provider; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import junit.framework.TestCase; + +import org.springframework.batch.io.exception.ValidationException; +import org.springframework.batch.io.file.FieldSet; +import org.springframework.batch.io.file.FieldSetInputSource; +import org.springframework.batch.io.file.FieldSetMapper; +import org.springframework.batch.io.file.support.DefaultFlatFileInputSource; +import org.springframework.batch.item.provider.DefaultFlatFileItemProvider; +import org.springframework.batch.item.validator.Validator; +import org.springframework.batch.restart.GenericRestartData; +import org.springframework.batch.restart.RestartData; +import org.springframework.batch.restart.Restartable; +import org.springframework.batch.statistics.StatisticsProvider; +import org.springframework.batch.support.PropertiesConverter; +import org.springframework.core.io.ByteArrayResource; + +/** + * Unit tests for {@link DefaultFlatFileItemProvider} + * + * @author Robert Kasanicky + */ +public class DefaultFlatFileItemProviderTests extends TestCase { + + public static String FOO = "foo"; + // object under test + private DefaultFlatFileItemProvider itemProvider = new DefaultFlatFileItemProvider(); + + // Input source + private DefaultFlatFileInputSource source; + + //mock mapper + private FieldSetMapper mapper; + + private List list = new ArrayList(); + + // create mock objects and inject them into data provider + protected void setUp() throws Exception { + source = new DefaultFlatFileInputSource(); + source.setResource(new ByteArrayResource("a,b".getBytes())); + mapper = new FieldSetMapper() { + public Object mapLine(FieldSet fs) { + return FOO; + } + }; + itemProvider.setSource(source); + itemProvider.setMapper(mapper); + assertTrue(Restartable.class.isAssignableFrom(DefaultFlatFileInputSource.class)); + assertTrue(FieldSetInputSource.class.isAssignableFrom(DefaultFlatFileInputSource.class)); + assertTrue(StatisticsProvider.class.isAssignableFrom(DefaultFlatFileInputSource.class)); + } + + /** + * Uses input template to provide the domain object. + */ + public void testNext() { + Object result = itemProvider.next(); + assertSame("domain object is provided by the input template", FOO, result); + } + + /** + * Uses input template to provide the domain object. + */ + public void testNextWithValidator() { + itemProvider.setValidator(new Validator() { + public void validate(Object value) throws ValidationException { + list.add(value); + } + }); + itemProvider.next(); + assertSame("domain object is provided by the input template", FOO, list.get(0)); + } + + /** + * Uses input template to provide the domain object. + */ + public void testNextWithValidatorAndInvalidData() { + itemProvider.setValidator(new Validator() { + public void validate(Object value) throws ValidationException { + throw new ValidationException("Invalid input"); + } + }); + try { + itemProvider.next(); + fail("Expected ValidationException"); + } catch (ValidationException e) { + // expected + assertEquals("Invalid input", e.getMessage()); + } + } + + /** + * Gets statistics from the input template + */ + public void testGetStatistics() { + Properties statistics = ((StatisticsProvider) source).getStatistics(); + assertEquals(statistics, itemProvider.getStatistics()); + } + + /** + * Gets statistics from the input template + */ + public void testGetStatisticsWithoutStatisticsProvider() { + itemProvider.setSource(null); + Properties props = itemProvider.getStatistics(); + assertEquals(null, props.getProperty("a")); + } + + /** + * Gets restart data from the input template + */ + public void testGetRestartData() { + RestartData data = ((Restartable) source).getRestartData(); + assertEquals(data.getProperties(), itemProvider.getRestartData().getProperties()); + } + + /** + * Forwarded restart data to input template + */ + public void testRestoreFrom() { + + final List list = new ArrayList(); + + RestartData data = new RestartData() { + + public Properties getProperties() { + list.add(FOO); + return ((Restartable) source).getRestartData().getProperties(); + }}; + + itemProvider.restoreFrom(data); + + //assertEquals(1, list.size()); getProperties are called multiple times due to null checks + assertTrue(list.size() > 0); + } + + /** + * Forward restart data to input template + * @throws Exception + */ + public void testRestoreFromWithoutRestartable() throws Exception { + itemProvider.setSource(null); + try { + itemProvider.restoreFrom(new GenericRestartData(PropertiesConverter.stringToProperties("value=bar"))); + fail("Expected IllegalStateException"); + } + catch (IllegalStateException e) { + // expected + } + } + + /** + * Forward restart data to input template + * @throws Exception + */ + public void testGetRestartDataWithoutRestartable() throws Exception { + itemProvider.setSource(null); + try { + itemProvider.getRestartData(); + fail("Expected IllegalStateException"); + } + catch (IllegalStateException e) { + // expected + } + } + + +} diff --git a/infrastructure/src/test/java/org/springframework/batch/item/provider/InputSourceItemProviderTests.java b/infrastructure/src/test/java/org/springframework/batch/item/provider/InputSourceItemProviderTests.java new file mode 100644 index 000000000..1157fece8 --- /dev/null +++ b/infrastructure/src/test/java/org/springframework/batch/item/provider/InputSourceItemProviderTests.java @@ -0,0 +1,123 @@ +/* + * Copyright 2006-2007 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.item.provider; + +import java.util.Properties; + +import junit.framework.TestCase; + +import org.springframework.batch.io.InputSource; +import org.springframework.batch.io.Skippable; +import org.springframework.batch.item.provider.InputSourceItemProvider; +import org.springframework.batch.restart.GenericRestartData; +import org.springframework.batch.restart.RestartData; +import org.springframework.batch.restart.Restartable; +import org.springframework.batch.statistics.StatisticsProvider; +import org.springframework.batch.support.PropertiesConverter; + +/** + * Unit test for {@link InputSourceItemProvider} + * + * @author Robert Kasanicky + */ +public class InputSourceItemProviderTests extends TestCase { + + // object under test + private InputSourceItemProvider itemProvider = new InputSourceItemProvider(); + + private InputSource source; + + // create input template and inject it to data provider + protected void setUp() throws Exception { + source = new MockInputSource(this); + itemProvider.setInputSource(source); + } + + /** + * Uses input template to provide the domain object. + */ + public void testNext() { + Object result = itemProvider.next(); + assertSame("domain object is provided by the input template", this, result); + } + + /** + * Gets statistics from the input template + */ + public void testGetStatistics() { + Properties props = itemProvider.getStatistics(); + assertEquals("b", props.getProperty("a")); + } + + /** + * Gets restart data from the input template + */ + public void testGetRestartData() { + Properties props = itemProvider.getRestartData().getProperties(); + assertEquals("foo", props.getProperty("value")); + } + + /** + * Forwared restart data to input template + */ + public void testRestoreFrom() { + itemProvider.restoreFrom(new GenericRestartData(PropertiesConverter.stringToProperties("value=bar"))); + assertEquals("bar", itemProvider.next()); + } + + public void testSkip() { + itemProvider.skip(); + assertEquals("after skip", itemProvider.next()); + } + + private class MockInputSource implements InputSource, StatisticsProvider, Restartable, Skippable { + + private Object value; + + public Properties getStatistics() { + return PropertiesConverter.stringToProperties("a=b"); + } + + public RestartData getRestartData() { + return new GenericRestartData(PropertiesConverter.stringToProperties("value=foo")); + } + + public void restoreFrom(RestartData data) { + value = data.getProperties().getProperty("value"); + } + + public MockInputSource(Object value) { + this.value = value; + } + + public Object read() { + return value; + } + + public void close() { + } + + public void open() { + } + + public void skip() { + value = "after skip"; + } + + } + +}