diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/support/AggregateItemReader.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/support/AggregateItemReader.java index a73cb1510..490b4b277 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/support/AggregateItemReader.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/support/AggregateItemReader.java @@ -24,15 +24,14 @@ import org.apache.commons.logging.LogFactory; import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.MarkFailedException; import org.springframework.batch.item.ResetFailedException; -import org.springframework.batch.item.file.mapping.FieldSetMapper; /** * An {@link ItemReader} that delivers a list as its item, storing up objects * from the injected {@link ItemReader} until they are ready to be packed out as - * a collection. The {@link ItemReader} should mark the beginning and end of - * records with the constant values in {@link FieldSetMapper} ( - * {@link AggregateItemReader#BEGIN_RECORD} and - * {@link AggregateItemReader#END_RECORD}).
+ * a collection. Usually this class will be used as a wrapper for a custom + * {@link ItemReader} that can identify the record boundaries. The custom reader + * should mark the beginning and end of records with the constant values ({@link AggregateItemReader#BEGIN_RECORD} + * and {@link AggregateItemReader#END_RECORD}).
* * This class is thread safe (it can be used concurrently by multiple threads) * as long as the {@link ItemReader} is also thread safe. @@ -124,7 +123,6 @@ public class AggregateItemReader implements ItemReader> { */ private class ResultHolder { List records = new ArrayList(); - boolean exhausted = false; } diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/support/AggregateItemReaderTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/support/AggregateItemReaderTests.java index fa04c7996..e5e0d695f 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/support/AggregateItemReaderTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/support/AggregateItemReaderTests.java @@ -1,61 +1,60 @@ package org.springframework.batch.item.support; import java.util.Collection; -import java.util.Iterator; import junit.framework.TestCase; -import org.easymock.MockControl; import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.support.AggregateItemReader; public class AggregateItemReaderTests extends TestCase { - private MockControl inputControl; private ItemReader input; + private AggregateItemReader provider; - @SuppressWarnings("unchecked") public void setUp() { + // create mock for input + input = new AbstractItemReader() { - //create mock for input - inputControl = MockControl.createControl(ItemReader.class); - input = (ItemReader) inputControl.getMock(); + private int count = 0; - //create provider + public Object read() { + switch (count++) { + case 0: + return AggregateItemReader.BEGIN_RECORD; + case 1: + case 2: + case 3: + return "line"; + case 4: + return AggregateItemReader.END_RECORD; + default: + return null; + } + } + + }; + // create provider provider = new AggregateItemReader(); provider.setItemReader(input); } public void testNext() throws Exception { - - //set-up mock input - input.read(); - inputControl.setReturnValue(AggregateItemReader.BEGIN_RECORD); - input.read(); - inputControl.setReturnValue("line",3); - input.read(); - inputControl.setReturnValue(AggregateItemReader.END_RECORD); - input.read(); - inputControl.setReturnValue(null); - inputControl.replay(); - - //read object + // read object Object result = provider.read(); - //it should be collection of 3 strings "line" + // it should be collection of 3 strings "line" assertTrue(result instanceof Collection); - Collection lines = (Collection)result; + Collection lines = (Collection) result; assertEquals(3, lines.size()); - for (Iterator i = lines.iterator(); i.hasNext();) { - assertEquals("line", i.next()); + for (Object line : lines) { + assertEquals("line", line); } - //read object again - it should return null + // read object again - it should return null assertNull(provider.read()); - //verify method calls - inputControl.verify(); } + } diff --git a/spring-batch-samples/src/main/java/org/springframework/batch/sample/domain/trade/internal/CustomerCreditUpdatePreparedStatementSetter.java b/spring-batch-samples/src/main/java/org/springframework/batch/sample/domain/trade/internal/CustomerCreditUpdatePreparedStatementSetter.java index b029f73ab..1f2ef6bd2 100644 --- a/spring-batch-samples/src/main/java/org/springframework/batch/sample/domain/trade/internal/CustomerCreditUpdatePreparedStatementSetter.java +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/domain/trade/internal/CustomerCreditUpdatePreparedStatementSetter.java @@ -29,6 +29,8 @@ import org.springframework.batch.sample.domain.trade.CustomerCredit; public class CustomerCreditUpdatePreparedStatementSetter implements ItemPreparedStatementSetter { public static final BigDecimal FIXED_AMOUNT = new BigDecimal(1000); + + public static final String QUERY = "UPDATE CUSTOMER SET CREDIT=? WHERE ID=?"; /* (non-Javadoc) * @see org.springframework.batch.io.support.ItemPreparedStatementSetter#setValues(java.lang.Object, java.sql.PreparedStatement) diff --git a/spring-batch-samples/src/main/resources/jobs/batchUpdateJob.xml b/spring-batch-samples/src/main/resources/jobs/batchUpdateJob.xml index 1f0bd8370..b6d98cb9e 100644 --- a/spring-batch-samples/src/main/resources/jobs/batchUpdateJob.xml +++ b/spring-batch-samples/src/main/resources/jobs/batchUpdateJob.xml @@ -1,11 +1,11 @@ + http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd"> Example for SQL Batch Update integration. @@ -25,7 +25,7 @@ - + diff --git a/spring-batch-samples/src/main/resources/jobs/beanWrapperMapperSampleJob.xml b/spring-batch-samples/src/main/resources/jobs/beanWrapperMapperSampleJob.xml index 98a1b12c4..9a6c0630f 100644 --- a/spring-batch-samples/src/main/resources/jobs/beanWrapperMapperSampleJob.xml +++ b/spring-batch-samples/src/main/resources/jobs/beanWrapperMapperSampleJob.xml @@ -1,84 +1,71 @@ - - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + diff --git a/spring-batch-samples/src/site/apt/index.apt b/spring-batch-samples/src/site/apt/index.apt index 4ab77577a..4f99a1e17 100644 --- a/spring-batch-samples/src/site/apt/index.apt +++ b/spring-batch-samples/src/site/apt/index.apt @@ -46,37 +46,43 @@ Spring Batch Samples Here is a list of samples with checks to indicate which features each one demonstrates: *----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ -|<> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> +|<> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | <> | *---- -adhocLoopJob | | | | | | | | | | | | | | | x | | +{{adhocLoop}} | | | | | | | | | | | | | | |x | | | | *---- -beanWrapperMapperSample | | x | | | | | | | | | x | | | x | | x | +{{batchUpdate}} | | | | | |x | | | | | | | | | | | |x | *---- -compositeProcessorSample | | | | | | | | x | | | x | | | | | x | +{{beanWrapperMapperSample}} | |x | | | | | | | | |x | | |x | |x | | | *---- -delegatingJob | | | | | | | | | | | | | | | | | x +{{compositeItemWriterSample}}| | | | | | | |x | | |x | | | | |x | | | *---- -fixedLengthImportJob | | x | | | | | | | | | x | | | | | x | +{{delegating}} | | | | | | | | | | | | | | | | |x | | *---- -hibernateJob | | | | | | x | | | | | x | | | | | | +{{fixedLengthImport}} | |x | | | | | | | | |x | | | | |x | | | *---- -ibatisJob | | | | | x | | | | | | x | | | | | | +{{football}} |x | | | | |x | | | | |x | | | | | | | | *---- -infiniteLoopJob | | | | | | | | | | | | | | | | | +{{hibernate}} | | | | | |x | | | | |x | | | | | | |x | *---- -multilineJob | | x | | x | | | | | | | | | | | | | +{{ibatis}} | | | | |x | | | | | |x | | | | | | | | *---- -multilineOrderJob | x | | | x | | | | x | | x | | | | | | | +{{multiline}} | |x | |x | | | | | | | | | | | | | | | *---- -fotballlJob | x | | | | | x | | | | | x | | | | | | +{{multilineOrder}} |x | | |x | | | |x | |x | | | | | | | | | *---- -restartSample | | x | | | | | | | | | x | | x | | | x | +{{quartzSample}} |x | | | | |x | | | | |x | | | |x | | | | *---- -simpleTaskletJob | | | | | | | | | | | | | | | | | +{{restartSample}} | |x | | | | | | | | |x | |x | | |x | | | *---- -tradeJob | x | | | | x | | x | | | | x | | | | | x | +{{retrySample}} | | | | | | | | | | | | | | | |x | | | *---- -xmlStaxJob | | | x | | | | | | x | | | | | | | | +{{skipSample}} |x | | | |x | |x | | | |x |x | | | |x | | | +*---- +{{tasklet}} | | | | | | | | | | | | | | | | | | | +*---- +{{trade}} |x | | | |x | |x | | | |x | | | | |x | | | +*---- +{{xmlStax}} | | |x | | | | | |x | | | | | | | | | | *---- * Common Sample Source Structures @@ -102,84 +108,93 @@ xmlStaxJob | | | x | | | | | | x | | | | | | | | Each job consists of several steps, these steps are defined in steps property. -** Tasklet Job +* Adhoc Loop and JMX Demo ({adhocLoop}) - The goal is to show the simplest use of the batch framework with a - single job with a single step, which cleans up a directory and runs - a system command. + This job is simply an infinite loop. It runs forever so it is + useful for testing features to do with stopping and starting jobs. + It is used, for instance, as one of the jobs that can be run from + JMX using the Eclipse launch configuration "jmxLauncher". - This job is defined by <<>> file. The - <<>> itself is defined by the bean definition with - <<>>. In this example we have two steps. + The JMX launcher uses an additional XML configuration file + (adhoc-job-launcher-context.xml) to set up a <<>> for + running jobs asynchronously (i.e. in a backgound thread). This + follows the same pattern as the {{{quartzSample}Quartz sample}}, so + see that section for more details of the <<>> + configuration. - * The first step defines a tasklet that is responsible for - clearing out a directory though a custom <<>>. Each - tasklet has an <<>> method which is called by the - step. All processing of business data should be handled by this - method. + The rest of the configuration for this demo consists of exposing + some components from the application context as JMX managed beans. + The <<>> is exposed as a stripped down interface + <<>>, so that it can be controlled from a + remote client (such as JConsole from the JDK) which does not have + Spring Batch on the classpath. See the Spring Core Reference Guide + for more details on how to customize the JMX configuration. - * The second step uses another tasklet to execute a system (OS) - command line. +* Batch Update ({batchUpdate}) - taskletJob.xml + The purpose of this sample is to show to usage of the + <<>> to make efficient updates to a + database table. - You can visualize the Spring configuration of a job through - Spring-IDE. See {{{http://springide.org/blog/}Spring IDE}}. The - source view of the configuration is as follows: + The <<>> accepts a special form of + <<>> as a (mandatory) dependency. This is + responsible for copying fields from the item to be written to a + <<>> matching the SQL query that has been + injected. The implementation of the + <<>> shows best + practice of keeping all the information needed for the execution in + one place, since it contains a static constant value (<<>>) + which is used to configure the query for the writer. -+--- - - - - - - - - - - - - - - - - - - - - - +* BeanWrapperMapper Sample ({beanWrapperMapperSample}) - + This sample shows the use of automatic mapping from fields in a file + to a domain object. The <<>> and <<>> objects needed + by the job are created from the Spring configuration using prototype + beans, and then their properties are set using the + <<>>, which sets properties of the + prototype according to the field names in the file. - - - -+--- + Nested property paths are resolved in the same way as normal Spring + binding occurs, but with a little extra leeway in terms of spelling + and capitalization. Thus for instance, the <<>> object has a + property called <<>> (lower case), but the file has been + configured to have a column name <<>> (upper case), and + the mapper will accept the values happily. Underscores instead of + camel-casing (e.g. <<>> instead of <<>>) + also work. - For simplicity we are only displaying the job configuration itself - and leaving out the details of the supporting batch execution - environment configuration. +* Composite ItemWriter Sample ({compositeItemWriterSample}) -** Fixed Length Import Job + This shows a common use case using a composite pattern, composing + instances of other framework readers or writers. It is also quite + common for business-specific readers or writers to wrap + off-the-shelf components in a similar way. + + In this job the composite pattern is used just to make duplicate + copies of the output data. The delegates for the + <<>> have to be separately registered as + streams in the <<>> where they are used, in order for the step + to be restartable. This is a common feature of all delegate + patterns. + +* Delegating Sample ({delegating}) + + This sample shows the delegate pattern again, and also the + <<>> which is used to adapt a POJO to the + <<>> interface. + +* Fixed Length Import Job ({fixedLengthImport}) The goal is to demonstrate a typical scenario of importing data from a fixed-length file to database - This job shows a more typical scenario, when reading - input data and processing the data is cleanly separated. The data - provider is responsible for reading input and mapping each record to - a domain object, which is then passed to the module processor. The - module processor handles the processing of the domain objects, in - this case it only writes them to database. - - fixedLengthImportJob.xml - - file with fixed row structure + This job shows a typical scenario, when reading input data and + processing the data is cleanly separated. The data provider is + responsible for reading input and mapping each record to a domain + object, which is then passed to the module processor. The module + processor handles the processing of the domain objects, in this case + it only writes them to database. In this example we are using a simple fixed length record structure that can be found in the project at @@ -214,109 +229,7 @@ xmlStaxJob | | | x | | | | | | x | | | | | | | | object -* Multiline Order Job - - The goal is to demostrate how to handle a more complex file input - format, where a record meant for processing inludes nested records - and spans multiple lines - - multilineOrderJob.xml - - file with multiline records. OrderDataProvider is - an example of a non-default programmatic data provider. It reads - input until it detects that the multiline record has finished and - encapsulates the record in a single domain object. - - file with multiline records. The concrete - <<>> passes the object to a an injected 'delegate - writer' which in this case writes the output to a file. The writer - in this case demonstrates how to write multiline output using a - custom aggregator transformer. - -* Quartz Sample - - The goal is to demonstrate how to schedule job execution using - Quartz scheduler. In this case there is no unit test to launch the - sample because it just re-uses the football job. There is a main - method in <<>> and an Eclipse launch - configuration which runs it with empty arguments. The main method - is very basic - it is intended only as a guide to how Quartz might - be used in principle. - - <<>>, also re-uses - <<>> - - The configuration declares a <<>> bean. The launcher - bean is different from the other samples only in that it uses an - asynchronous task executor, so that the jobs are launched in a - separate thread to the main method: - -+--- - - - - - - -+--- - - Also, a Quartz <<>> is defined using a Spring - <<>> as a convenience. - -+-- - - - - - - - - -+-- - - Finally, a trigger with a scheduler is defined that will launch the - job detail every 10 seconds: - -+--- - - - - - - - - -+--- - - The job is thus scheduled to run every 10 seconds. In fact it - should be successful on the first attempt, so the second and - subsequent attempts should through a - <<>>. In a production system, - the job detail would probably be modified to account for this - exception (e.g. catch it and re-submit with a new set of job - parameters). The point here is that Spring Batch guarantees that - the job execution is idempotent - you can never inadvertently - process the same data twice. - -* Trade Job - - The goal is to show a reasonably complex scenario, that would - resemble the real-life usage of the framework. - - This job has 3 steps. First, data about trades are - imported from a file to database. Second, the trades are read from - the database and credit on customer accounts is decreased - appropriately. Last, a report about customers is exported to a file. - - <<>> - the job definition, - <<>> - input and output configuration - - This job has 3 steps. First, data about trades is - imported from a file to database. Second, the data about trades is - read from the database and credit on customer accounts is decreased - appropriately. Last, a report about customers is exported to a file. - -* Football Job +* Football Job ({football}) This is a (American) Football statistics loading job. We gave it the id of <<>> in our configuration file. Before diving @@ -448,7 +361,7 @@ AbduKa00,1996,mia,15,nyg,0,0,0,0,0,17,96,,14,0 the developer can remain solely concerned with their business logic. - * – the item reader is the source of the information + * – the item reader is the source of the information pipe. At the most basic level input is read in from an input source, parsed into a domain object and returned. In this way, the good batch architecture practice of ensuring all data has been @@ -615,3 +528,200 @@ games.player_id group by games.player_id, games.year_no each of the records as they are processed. Please keep in mind that AoP is used to wrap the <<>> and output each record as it is processed to the logger, which may impact performance. + +* Hibernate {hibernate} + + The purpose of this sample is to show a typical usage of Hibernate + as an ORM tool in the input and output of a job. + + The job uses a <<>> for the input, where + a simple HQL query is used to supply items. It also uses a + non-framework <<>> wrapping a DAO, which perhaps was + written as part of an online system. + + + The output reliability and robustness are improved by the use of the + <<>> from the framework. One of its roles + is to buffer items and flush them explicitly, rather than implicitly + on a transaction boundary (which would be the default). This + "write-behind" behaviour is provided by Hibernate implicitly, but we + need to take control of it so that the skip and retry features + fprovided by Spring Batch can work effectively. Thus the other role + of the <<>> is to watch out for failures + and flush aggressively when an item is seen from a previously failed + chunk. In this way there will always be a failure at some point + immediately after the bad item was written, and the item is then + easily identifiable. + +* Ibatis ({ibatis}) + + The goal of this sample is to show the use of Ibatis as a query + mapping tool. Its features are similar to the Hibernate sample, but + it uses Ibatis to drive its input and output. + +* Multiline ({multiline}) + +* Multiline Order Job ({multilineOrder}) + + The goal is to demostrate how to handle a more complex file input + format, where a record meant for processing inludes nested records + and spans multiple lines + + The input source is file with multiline records. + <<>> is an example of a non-default programmatic + item reader. It reads input until it detects that the multiline + record has finished and encapsulates the record in a single domain + object. + + The output target is a file with multiline records. The concrete + <<>> passes the object to a an injected 'delegate + writer' which in this case writes the output to a file. The writer + in this case demonstrates how to write multiline output using a + custom aggregator transformer. + +* Quartz Sample ({quartz}) + + The goal is to demonstrate how to schedule job execution using + Quartz scheduler. In this case there is no unit test to launch the + sample because it just re-uses the football job. There is a main + method in <<>> and an Eclipse launch + configuration which runs it with arguments to pick up the football + job. + + The additional XML configuration for this job is in + <<>>, and it also re-uses + <<>> + + The configuration declares a <<>> bean. The launcher + bean is different from the other samples only in that it uses an + asynchronous task executor, so that the jobs are launched in a + separate thread to the main method: + ++--- + + + + + + ++--- + + Also, a Quartz <<>> is defined using a Spring + <<>> as a convenience. + ++-- + + + + + + + + + + + ++-- + + Finally, a trigger with a scheduler is defined that will launch the + job detail every 10 seconds: + ++--- + + + + + + + + ++--- + + The job is thus scheduled to run every 10 seconds. In fact it + should be successful on the first attempt, so the second and + subsequent attempts should through a + <<>>. In a production system, + the job detail would probably be modified to account for this + exception (e.g. catch it and re-submit with a new set of job + parameters). The point here is that Spring Batch guarantees that + the job execution is idempotent - you can never inadvertently + process the same data twice. + +* Restart Sample ({restartSample}) + +* Retry Sample ({retrySample}) + +* Skip Sample ({skipSample}) + +* Tasklet Job ({tasklet}) + + The goal is to show the simplest use of the batch framework with a + single job with a single step, which cleans up a directory and runs + a system command. + + The + <<>> itself is defined by the bean definition with + <<>>. In this example we have two steps. + + * The first step defines a tasklet that is responsible for + clearing out a directory though a custom <<>>. Each + tasklet has an <<>> method which is called by the + step. All processing of business data should be handled by this + method. + + * The second step uses another tasklet to execute a system (OS) + command line. + + You can visualize the Spring configuration of a job through + Spring-IDE. See {{{http://springide.org/blog/}Spring IDE}}. The + source view of the configuration is as follows: + ++--- + + + + + + + + + + + + + + + + + + + + + + + + + + + ++--- + + For simplicity we are only displaying the job configuration itself + and leaving out the details of the supporting batch execution + environment configuration. + +* Trade Job ({trade}) + + The goal is to show a reasonably complex scenario, that would + resemble the real-life usage of the framework. + + This job has 3 steps. First, data about trades are + imported from a file to database. Second, the trades are read from + the database and credit on customer accounts is decreased + appropriately. Last, a report about customers is exported to a file. + +* XML Input Output ({xmlStax})