diff --git a/docs/models/Figures.ppt b/docs/models/Figures.ppt index 983683c5a..bcb2bd613 100644 Binary files a/docs/models/Figures.ppt and b/docs/models/Figures.ppt differ diff --git a/docs/pom.xml b/docs/pom.xml index 28e12ed85..bb2eb4c1f 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -19,6 +19,9 @@ agilejava http://agilejava.com/maven + + false + @@ -35,17 +38,19 @@ generate-html - false ${basedir}/src/docbkx/resources/xsl/html.xsl - + - + + + + @@ -60,23 +65,34 @@ pre-site - multi-page + single-pdf generate-pdf + + + src/site/docbook/reference/ + src/docbkx/resources/images/ + + + + + pre-site + + + multi-page + generate-html true - ${basedir}/src/site/docbook/reference/ ${basedir}/src/docbkx/resources/xsl/html_chunk.xsl - - + - + @@ -105,7 +121,7 @@ runtime - + org.apache.xmlgraphics fop 0.93 @@ -113,10 +129,12 @@ index.xml + false + css/html.css + ${basedir}/src/site/docbook/reference ${basedir}/src/docbkx/resources/xsl/fopdf.xsl true - ${basedir}/src/site/docbook/reference version diff --git a/docs/src/docbkx/resources/images/important.png b/docs/src/docbkx/resources/images/important.png new file mode 100644 index 000000000..ad57f6f72 Binary files /dev/null and b/docs/src/docbkx/resources/images/important.png differ diff --git a/docs/src/docbkx/resources/images/note.png b/docs/src/docbkx/resources/images/note.png new file mode 100644 index 000000000..ad57f6f72 Binary files /dev/null and b/docs/src/docbkx/resources/images/note.png differ diff --git a/docs/src/docbkx/resources/images/tip.png b/docs/src/docbkx/resources/images/tip.png new file mode 100644 index 000000000..ad57f6f72 Binary files /dev/null and b/docs/src/docbkx/resources/images/tip.png differ diff --git a/docs/src/models/domain-chunck-view-classdiagram.gif b/docs/src/models/domain-chunck-view-classdiagram.gif deleted file mode 100644 index 9417acfd8..000000000 Binary files a/docs/src/models/domain-chunck-view-classdiagram.gif and /dev/null differ diff --git a/docs/src/models/domain-classdiagram.dnx b/docs/src/models/domain-classdiagram.dnx deleted file mode 100644 index 8a3a3af86..000000000 --- a/docs/src/models/domain-classdiagram.dnx +++ /dev/null @@ -1,847 +0,0 @@ - - -?> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/src/models/flat-file-input-source-design.dnx b/docs/src/models/flat-file-input-source-design.dnx deleted file mode 100644 index b18b0180a..000000000 --- a/docs/src/models/flat-file-input-source-design.dnx +++ /dev/null @@ -1,551 +0,0 @@ - - -?> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/src/models/io-design.dnx b/docs/src/models/io-design.dnx deleted file mode 100644 index f5294f773..000000000 --- a/docs/src/models/io-design.dnx +++ /dev/null @@ -1,1392 +0,0 @@ - - -?> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/src/models/item-reader-design.dnx b/docs/src/models/item-reader-design.dnx deleted file mode 100644 index 0a981fb08..000000000 --- a/docs/src/models/item-reader-design.dnx +++ /dev/null @@ -1,983 +0,0 @@ - - -?> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/src/models/item-stream-adapter.dnx b/docs/src/models/item-stream-adapter.dnx deleted file mode 100644 index 62cb870ec..000000000 --- a/docs/src/models/item-stream-adapter.dnx +++ /dev/null @@ -1,353 +0,0 @@ - - -?> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/src/models/repository-classdiagram.dnx b/docs/src/models/repository-classdiagram.dnx deleted file mode 100644 index 4677d6176..000000000 --- a/docs/src/models/repository-classdiagram.dnx +++ /dev/null @@ -1,482 +0,0 @@ - - -?> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/src/site/docbook/reference/arch-overview.xml b/docs/src/site/docbook/reference/arch-overview.xml index b928a11c6..954f9adf5 100644 --- a/docs/src/site/docbook/reference/arch-overview.xml +++ b/docs/src/site/docbook/reference/arch-overview.xml @@ -1,546 +1,546 @@ - - - - Container Architecture Overview -
- Introduction - -
- -
- Simple Container Architecture Overview - -
- - Introduction - - This chapter covers the overall spring batch architecture. - The Spring Container Archtiecture is made up of five logical - layers; 1) the Batch Application, 2) the Batch Application Layer, - 3) the batch core layer, and 4) the batch infrastucture - layer. - - - - - - - - - - - Provided - ByLayerDescription - - Application - DeveloperBatch - ApplicationThis is where the - application developer writes their batch jobs and - tasklets. - - Spring Batch Execution - ContainerContainer Application - LayerAllows for extending and - overwriting of the batch support layer for custom - requirements. Facilities implemented in this layer could - migrate down to Batch Support Layer. This is also the layer - to add the project specific jars required by job types - (e.g. reporting jars like Crystal, Brio, etc, form - generation jars like Central Pro or Adobe, - etc). - - Spring Batch Execution - ContainerContainer Support - LayerProvides default - implementations of batch core services including I/O, - Restart, Partitioning, Statistics, and - configurations - - Spring Batch Execution - ContainerContainer Core - LayerEnables configuration, - Common Services & Interfaces, - management - - Spring Batch - InfrastructureBatch-InfrastructureProvides - IO support, Batch style transactions, advanced exception - handling, batch-template, batch-retry - - - - - - - - - - - - Figure 2.0 - - - - - Batch Architecture Layers - - The batch architecture is modeled after a container - architecture, meaning that there are managed resources - essential to high performance batch architectures that are - configured through a spring context. The following sect1s - will provide a quick review of each layer and their role in - the batch architecture. - - - - - - - -
- -
- - Batch Applications - - -
- -
- - Container Application Layer - - -
- -
- - Container Support Layer - - The batch support layer provides default implementations - for all interfaces, interceptors, advice and other core batch - services. Figure 2.3.1 illustrates the following logical - packages. !Batch Support.png! Although physically they break out - into many more than depicted, logically you can think of the - groupings in the following manner: * I/O Support packages * - Restart Support * Lifecycle Support packages * DAO support - layer - - - - - - I/O Support Packages - - The I/O related packages are currently the richest - packages in the batch architecture. They are modeled after - Spring Patterns of Operations and Templates. For example, - you'll see FlatFileInputOperations accompanied with a - FlatFileInputTemplate. The FlatFileInputTemplate is wired up - with a File Descriptor, which contains a Record Descriptor - along with various other properties. With the File and Record - Descriptors the InputTemplate supports a callback method that - allows for the mapping of a record into an object. This - support applies to fixed length records, delimited records - and XML records. To further simplify this a - DefaultFlatFileDataProvider is supplied an input template, - which contains the field and record descriptions, along with - a line mapper that knows how to map the line to an object. - The next() operation on a record simply needs to - readAndMap(lineMapper) a record. This pattern is used over - again for XML and SQL input for simple mapping of input - records to objects. - - In addition to declarative descriptions of the records - that can be re-used by multiple batch jobs, the I/O - facilities also support configurable validation strategies. - The two currently supported are Apache Commons Validator and - Spring's VALang. - - - - - - Restart Support - - The Restart Support provides implementations for a few - common restart strategies that will be discussed further in - the respective sect1. The following are provided - out-of-the-box: * IDList Restart Strategy - a strategy that - supports a batch application where the application does not - have a "process" flag and needs the batch - architecture to track which records have been processed. This - is not the ideal scenario. * Last Processed Restart Strategy - - when the record can be identified through a where and order - by only the last record(s) processed needs to be saved for - restart. * No Restart Strategy - some batch jobs simply - can't support restart. When they are re-run they are - considered to be a new instance of a batch job. * Sql Restart - Strategy - [need some additional javadoc for this - strategy]. - - - - - - Lifecycle Support - - - - - -
-
-
- - The Core Layer - - The Batch Core interfaces and services are illustrated in - a simplified view of a package diagram. There are roughly seven - logical packages: * Core Spring Extensions * Core Batch Advice * - Core Batch Configuration * Core Batch Repository * Core Batch - Tasklet \ !Batch Core.png! [Figure 2.5] Batch Core Layer - - In the actual physical packaging there are a few more - logical services that the batch execution environment provides. The following - sections will provide an introduction into each set of core batch - facilities. - - - - - - Core Spring Extensions - - The Core Spring extensions provide the scaffolding for - a batch execution environment. This includes facilities for managing the - batch architecture in terms of launching, suspending and - stopping batch jobs. There is house keeping that goes on, - especially in concurrent batch jobs, related to ensuring that - batch jobs quiese properly. The lifecycle management provides - services for the proper initialization and subsequent - shutdown of batch resources and services. The batch - architecture is flexible in terms of how batch jobs may be - launched. For example, batch jobs can be started via JMX - facilities, scripts from the command line that launch a Java - VM. It can also support launching batch jobs through web - services or http. There are no restrictions. Finally, there - are standard batch error codes. These error codes can be - exposed to external utilities, like Schedulers, to ensure - that batch jobs expose the status of jobs to an operational - environment. This is especially important in the batch - context where the modus operandi is headless, meaning - unattended operation. - - - - - - Core Batch Advice - - Core Batch Advice is an inventory of the type of advice - that batch architectures will inject during the runtime of a - batch application. These are defined as a set of extensible - interfaces, with a number of default implementations in the - support layer that provide some of the most common types of - advice. Partition Advice is helpful with large datasets that - need to be "chunked" up and run concurrently for - better through put. Resource Advice is helpful for - registering interest in transactional information so that - file locations can be kept in sync with information processed - within a transaction. In addition, the resource is associated - with the correct step context and its associated - configuration properties. Skip advice is applied for records - that the tasklet is unable to process. Restart Advice is - helpful for Restartable jobs where for advising the job on - how to restart. There is considerable variability on how - restart can occur. For example, a job may be marking records - as "processed" and the restart advice will advise - the process with query that restarts the job at the last - successfully processed record. Finally, Statistics are vital - in operational environments to report on records processed, - records skipped and total number of records read. In - addition, certain batch jobs lend themselves to custom - reporting to expose additional business level information - like the number of trades processed or cases opened, - etc. - - - - - - Core Batch Configuration - - Batch configuration is considerably different from - online web applications or SOA based applications. The Core - Batch Configuration provides a place for configuring runtime - properties related to the batch application style. This - includes the ability to add Commit Policy. In a batch style - application it is often advantageous to keep the commit - interval as high as possible when processing Logical Units of - Work. Whereas in an online web application with declarative - transaction the transaction scope would be at the entrance to - a business service, a batch transaction scope may include - many logical units of work before a transaction commit is - executed. A Start Policy allows a configuration to tell the - batch job whether it is Restartable, and if so, what type of - restart to initiate. Some jobs are not restartable and care - should be taken to ensure that information is not applied - multiple times when the business rules do not allow for it. - Exception policies deal with what to do when exceptions - occur. This impacts logging policies and exception handling. - The architecture defines a common set of exceptions that - projects can apply handlers to like processing errors, - validation errors, parsing errors, missing configuration - parameters, etc. - - - - - - 2.5.4 Container Repository - - This is an internal package for storing the state of a - batch job and any associated partition and step status. - - - - - - Core Batch Tasklet - - The core batch tasklet is where control is handed off to - the application. There are a number of patterns that have - been observed in processing batch data. Spring Core Batch - Tasklet implements the most common patterns and provides and - extension point for additional Tasklet processing - implementations. The basic idea of tasklet provides the - facilities for reading and processing data. The simplest - implementation of Tasklet, the ReadProcessTasklet, handles both - the input and output of data within one class. An alternative - implementation, the DataProviderProcessTasklet, provides - functionality for 'split processing'. This type of - processing is characterized by separating the reading and - processing of batch data into two seperate classes: - DataProvider and TaskletProcessor. The DataProvider class - provides a solid means for reusablility and enforces good - architecture practices. Because an object *must* be returned - by the DataProider to continue processing, (Returning null - indicates processing should end) a developer is forced to - read in all relevant data, place it into domain or value - objects, and return the object. The TaskletProcessor will then - use this object within the business logic and final - output. - - - - - -
- -
- - Container's Use of batch - infrastructure - - - - - - Infrastructure Provided I/O - - The I/O core interfaces and implementations provide - facilities for simplifying the extraction of data from I/O - sources like files and database tables. The key concepts are - FieldDescriptors and FieldSets along with appropriate CallBack - Handlers. These are modeled after common spring operations and - templates like JdbcTemplate. Through the use of LineMappers a - developer needs only to describe a record format and write the - appropriate callback method that maps the parsed record into an - object of their choice. These can either be true POJO objects - or Value Objects (structures) that are subsequently available - for the tasklet to processs. The interface for Field Descriptors - also allows for a level of validation through the use of - Spring's VALang or Apache's Common Validator. - - - - - - Core Batch Interceptors & Interceptor - Services - - - - - - Batch Operations & Batch Template - - Interceptors and the associated services are the key - to how advise is applied in the batch architecture. The - interceptors are Point Cuts in the batch lifecycle that - allow the injection of advise. The shared lifecycle - behavior abstracted through the BatchLifeCycleInterceptor - defineds three methods; init, onError and finalize. All - subclasses of LifeCycleInterceptor define default behavior - for these three methods. The JobLifecycleInterceptor - further exposes the methods beforeJob(), beforeStep(), - afterJob(), and afterStep() allowing hooks into the - lifecycle for specific advise. The Batch Architecture - provides default implementations for all lifecycle point - cuts, or interception points. The Tasklet Interceptor, in - addition to the standard lifecycle methods, implements - logic around beforeLuw(), afterLuw(), - commitIntervalStarted() and commitIntervalCompleted(). - Having well defined lifecycle interception points allows - for the easy insertion of custom advice into the batch - runtime environment. - - - - - - - - - -
- -
- - Batch Execution Container Configurations - - In addition to core facilities for configuring or wiring - together jobs and steps with required resources, policies, and - interceptors, spring batch allows considerable flexibility in how - scalability is achieved. More options for scalability will be - available in the future. The important key for scalability in Java - is the recognition that there is a limit to what one JVM may scale - up to in terms of number of threads, managed resources, memory - configuration, etc. The spring batch architecture allows for the - configuration of simple batch jobs where one VM and one process is - sufficient to do perform the work within a batch window all the way - through many threads distributed within a cluster of JEE servers. - The figure below illistrates the scalability spectrum. - - This is not to be understood as the only way to scale batch - jobs as there are many factors. For example, other federated java - architectures hold potential like Teracotta or Gigaspaces although - there is no current implementation for these distributed models in - the current batch architecture. !scalability-model.png! [Figure - 2.3.1] - Scalability Model - -
- - Single VM Simple Batch Execution - Container - One Job, One Step, One Partition - - The simplest configuration is one job with with step and - hence, one implied partition. Implied means that there is nothing - for the developer to consider because the default number of - partitions is one. There is typically one input source and one - output source in this simple configuration. See the Simple Tasklet - Job for an example of what this configuration looks like. A - simple configuration still typically configures a datasource - context, the batch configuration for describing the Job, Step, - along with the associated configured policies, field descriptors, - and line mappers. !SimpleTradeConfiguration.jpg! [Figure 2.3.1] - Simple Container Configuration - - The details of this configuration will be covered - thoroughly in subsequent sect1s of the document but for now it - should be understood that Job, the Step, the input template, the - file descriptor with its associated line mapper, and the output - (e.g. the TradeWriter). - -
- -
- - Single VM Multi-threaded Batch Execution - Container Configuration - One Job, One Step, Multiple - Partitions - - In a Single JVM using partitioning a multi-threaded - execution is supported. [This is still work in progress] - -
- -
- - Batch Execution Container Hosted in J2EE - Container - managed environment - - The J2EE container model has fallen under fire over the - past few years for many valid reasons. There are some things that - the J2EE container do very well though that projects should - consider when planning for scalability with batch architectures. - Commercial and open source containers like WebSphere, BEA and - JBOSS typically: - - - - - - manage datasources effectively along with attendent - services like prepared statement caching. - - - - - - manage transactions effectively including many - configurable properties for long lived transactions. - - - - - - manage thread pools more effectively. - - - - - - supply robust implementations of JTA, a requirement - when batch jobs output to multiple XA resources like JMS and - JDBC. - - - - - - manage distribution effectively including domains, - clusters and cells - - - - - - provide robust JMX management for configuring, managing - and administering distributed applications. - - - - - - workload management facilities (clusters) provided by - J2EE vendors - - - - - - Projects are encouraged to deploy batch applications with - the simplest configuration possible, but when federated JVMs are - a requirement to process volumes of data within a batch window, - batch-in-container provides an effective way of distributing the - processing. Spring Batch supports this through a simple change in - configuration. [Work in progress on the exact implementation - - being released as part of M2]. - -
-
- - - + + + + Container Architecture Overview +
+ Introduction + +
+ +
+ Simple Container Architecture Overview + +
+ + Introduction + + This chapter covers the overall spring batch architecture. + The Spring Container Archtiecture is made up of five logical + layers; 1) the Batch Application, 2) the Batch Application Layer, + 3) the batch core layer, and 4) the batch infrastucture + layer. + + + + + + + + + + + Provided + ByLayerDescription + + Application + DeveloperBatch + ApplicationThis is where the + application developer writes their batch jobs and + tasklets. + + Spring Batch Execution + ContainerContainer Application + LayerAllows for extending and + overwriting of the batch support layer for custom + requirements. Facilities implemented in this layer could + migrate down to Batch Support Layer. This is also the layer + to add the project specific jars required by job types + (e.g. reporting jars like Crystal, Brio, etc, form + generation jars like Central Pro or Adobe, + etc). + + Spring Batch Execution + ContainerContainer Support + LayerProvides default + implementations of batch core services including I/O, + Restart, Partitioning, Statistics, and + configurations + + Spring Batch Execution + ContainerContainer Core + LayerEnables configuration, + Common Services & Interfaces, + management + + Spring Batch + InfrastructureBatch-InfrastructureProvides + IO support, Batch style transactions, advanced exception + handling, batch-template, batch-retry + + + + + + + + + + + + Figure 2.0 + + + + - Batch Architecture Layers + + The batch architecture is modeled after a container + architecture, meaning that there are managed resources + essential to high performance batch architectures that are + configured through a spring context. The following sect1s + will provide a quick review of each layer and their role in + the batch architecture. + + + + + + + +
+ +
+ + Batch Applications + + +
+ +
+ + Container Application Layer + + +
+ +
+ + Container Support Layer + + The batch support layer provides default implementations + for all interfaces, interceptors, advice and other core batch + services. Figure 2.3.1 illustrates the following logical + packages. !Batch Support.png! Although physically they break out + into many more than depicted, logically you can think of the + groupings in the following manner: * I/O Support packages * + Restart Support * Lifecycle Support packages * DAO support + layer + + + + + + I/O Support Packages + + The I/O related packages are currently the richest + packages in the batch architecture. They are modeled after + Spring Patterns of Operations and Templates. For example, + you'll see FlatFileInputOperations accompanied with a + FlatFileInputTemplate. The FlatFileInputTemplate is wired up + with a File Descriptor, which contains a Record Descriptor + along with various other properties. With the File and Record + Descriptors the InputTemplate supports a callback method that + allows for the mapping of a record into an object. This + support applies to fixed length records, delimited records + and XML records. To further simplify this a + DefaultFlatFileDataProvider is supplied an input template, + which contains the field and record descriptions, along with + a line mapper that knows how to map the line to an object. + The next() operation on a record simply needs to + readAndMap(lineMapper) a record. This pattern is used over + again for XML and SQL input for simple mapping of input + records to objects. + + In addition to declarative descriptions of the records + that can be re-used by multiple batch jobs, the I/O + facilities also support configurable validation strategies. + The two currently supported are Apache Commons Validator and + Spring's VALang. + + + + + + Restart Support + + The Restart Support provides implementations for a few + common restart strategies that will be discussed further in + the respective sect1. The following are provided + out-of-the-box: * IDList Restart Strategy - a strategy that + supports a batch application where the application does not + have a "process" flag and needs the batch + architecture to track which records have been processed. This + is not the ideal scenario. * Last Processed Restart Strategy + - when the record can be identified through a where and order + by only the last record(s) processed needs to be saved for + restart. * No Restart Strategy - some batch jobs simply + can't support restart. When they are re-run they are + considered to be a new instance of a batch job. * Sql Restart + Strategy - [need some additional javadoc for this + strategy]. + + + + + + Lifecycle Support + + + + + +
+
+
+ + The Core Layer + + The Batch Core interfaces and services are illustrated in + a simplified view of a package diagram. There are roughly seven + logical packages: * Core Spring Extensions * Core Batch Advice * + Core Batch Configuration * Core Batch Repository * Core Batch + Tasklet \ !Batch Core.png! [Figure 2.5] Batch Core Layer + + In the actual physical packaging there are a few more + logical services that the batch execution environment provides. The following + sections will provide an introduction into each set of core batch + facilities. + + + + + + Core Spring Extensions + + The Core Spring extensions provide the scaffolding for + a batch execution environment. This includes facilities for managing the + batch architecture in terms of launching, suspending and + stopping batch jobs. There is house keeping that goes on, + especially in concurrent batch jobs, related to ensuring that + batch jobs quiese properly. The lifecycle management provides + services for the proper initialization and subsequent + shutdown of batch resources and services. The batch + architecture is flexible in terms of how batch jobs may be + launched. For example, batch jobs can be started via JMX + facilities, scripts from the command line that launch a Java + VM. It can also support launching batch jobs through web + services or http. There are no restrictions. Finally, there + are standard batch error codes. These error codes can be + exposed to external utilities, like Schedulers, to ensure + that batch jobs expose the status of jobs to an operational + environment. This is especially important in the batch + context where the modus operandi is headless, meaning + unattended operation. + + + + + + Core Batch Advice + + Core Batch Advice is an inventory of the type of advice + that batch architectures will inject during the runtime of a + batch application. These are defined as a set of extensible + interfaces, with a number of default implementations in the + support layer that provide some of the most common types of + advice. Partition Advice is helpful with large datasets that + need to be "chunked" up and run concurrently for + better through put. Resource Advice is helpful for + registering interest in transactional information so that + file locations can be kept in sync with information processed + within a transaction. In addition, the resource is associated + with the correct step context and its associated + configuration properties. Skip advice is applied for records + that the tasklet is unable to process. Restart Advice is + helpful for Restartable jobs where for advising the job on + how to restart. There is considerable variability on how + restart can occur. For example, a job may be marking records + as "processed" and the restart advice will advise + the process with query that restarts the job at the last + successfully processed record. Finally, Statistics are vital + in operational environments to report on records processed, + records skipped and total number of records read. In + addition, certain batch jobs lend themselves to custom + reporting to expose additional business level information + like the number of trades processed or cases opened, + etc. + + + + + + Core Batch Configuration + + Batch configuration is considerably different from + online web applications or SOA based applications. The Core + Batch Configuration provides a place for configuring runtime + properties related to the batch application style. This + includes the ability to add Commit Policy. In a batch style + application it is often advantageous to keep the commit + interval as high as possible when processing Logical Units of + Work. Whereas in an online web application with declarative + transaction the transaction scope would be at the entrance to + a business service, a batch transaction scope may include + many logical units of work before a transaction commit is + executed. A Start Policy allows a configuration to tell the + batch job whether it is Restartable, and if so, what type of + restart to initiate. Some jobs are not restartable and care + should be taken to ensure that information is not applied + multiple times when the business rules do not allow for it. + Exception policies deal with what to do when exceptions + occur. This impacts logging policies and exception handling. + The architecture defines a common set of exceptions that + projects can apply handlers to like processing errors, + validation errors, parsing errors, missing configuration + parameters, etc. + + + + + + 2.5.4 Container Repository + + This is an internal package for storing the state of a + batch job and any associated partition and step status. + + + + + + Core Batch Tasklet + + The core batch tasklet is where control is handed off to + the application. There are a number of patterns that have + been observed in processing batch data. Spring Core Batch + Tasklet implements the most common patterns and provides and + extension point for additional Tasklet processing + implementations. The basic idea of tasklet provides the + facilities for reading and processing data. The simplest + implementation of Tasklet, the ReadProcessTasklet, handles both + the input and output of data within one class. An alternative + implementation, the DataProviderProcessTasklet, provides + functionality for 'split processing'. This type of + processing is characterized by separating the reading and + processing of batch data into two seperate classes: + DataProvider and TaskletProcessor. The DataProvider class + provides a solid means for reusablility and enforces good + architecture practices. Because an object *must* be returned + by the DataProider to continue processing, (Returning null + indicates processing should end) a developer is forced to + read in all relevant data, place it into domain or value + objects, and return the object. The TaskletProcessor will then + use this object within the business logic and final + output. + + + + + +
+ +
+ + Container's Use of batch + infrastructure + + + + + + Infrastructure Provided I/O + + The I/O core interfaces and implementations provide + facilities for simplifying the extraction of data from I/O + sources like files and database tables. The key concepts are + FieldDescriptors and FieldSets along with appropriate CallBack + Handlers. These are modeled after common spring operations and + templates like JdbcTemplate. Through the use of LineMappers a + developer needs only to describe a record format and write the + appropriate callback method that maps the parsed record into an + object of their choice. These can either be true POJO objects + or Value Objects (structures) that are subsequently available + for the tasklet to processs. The interface for Field Descriptors + also allows for a level of validation through the use of + Spring's VALang or Apache's Common Validator. + + + + + + Core Batch Interceptors & Interceptor + Services + + + + + + Batch Operations & Batch Template + + Interceptors and the associated services are the key + to how advise is applied in the batch architecture. The + interceptors are Point Cuts in the batch lifecycle that + allow the injection of advise. The shared lifecycle + behavior abstracted through the BatchLifeCycleInterceptor + defineds three methods; init, onError and finalize. All + subclasses of LifeCycleInterceptor define default behavior + for these three methods. The JobLifecycleInterceptor + further exposes the methods beforeJob(), beforeStep(), + afterJob(), and afterStep() allowing hooks into the + lifecycle for specific advise. The Batch Architecture + provides default implementations for all lifecycle point + cuts, or interception points. The Tasklet Interceptor, in + addition to the standard lifecycle methods, implements + logic around beforeLuw(), afterLuw(), + commitIntervalStarted() and commitIntervalCompleted(). + Having well defined lifecycle interception points allows + for the easy insertion of custom advice into the batch + runtime environment. + + + + + + + + + +
+ +
+ + Batch Execution Container Configurations + + In addition to core facilities for configuring or wiring + together jobs and steps with required resources, policies, and + interceptors, spring batch allows considerable flexibility in how + scalability is achieved. More options for scalability will be + available in the future. The important key for scalability in Java + is the recognition that there is a limit to what one JVM may scale + up to in terms of number of threads, managed resources, memory + configuration, etc. The spring batch architecture allows for the + configuration of simple batch jobs where one VM and one process is + sufficient to do perform the work within a batch window all the way + through many threads distributed within a cluster of JEE servers. + The figure below illistrates the scalability spectrum. + + This is not to be understood as the only way to scale batch + jobs as there are many factors. For example, other federated java + architectures hold potential like Teracotta or Gigaspaces although + there is no current implementation for these distributed models in + the current batch architecture. !scalability-model.png! [Figure + 2.3.1] - Scalability Model + +
+ + Single VM Simple Batch Execution + Container - One Job, One Step, One Partition + + The simplest configuration is one job with with step and + hence, one implied partition. Implied means that there is nothing + for the developer to consider because the default number of + partitions is one. There is typically one input source and one + output source in this simple configuration. See the Simple Tasklet + Job for an example of what this configuration looks like. A + simple configuration still typically configures a datasource + context, the batch configuration for describing the Job, Step, + along with the associated configured policies, field descriptors, + and line mappers. !SimpleTradeConfiguration.jpg! [Figure 2.3.1] + Simple Container Configuration + + The details of this configuration will be covered + thoroughly in subsequent sect1s of the document but for now it + should be understood that Job, the Step, the input template, the + file descriptor with its associated line mapper, and the output + (e.g. the TradeWriter). + +
+ +
+ + Single VM Multi-threaded Batch Execution + Container Configuration - One Job, One Step, Multiple + Partitions + + In a Single JVM using partitioning a multi-threaded + execution is supported. [This is still work in progress] + +
+ +
+ + Batch Execution Container Hosted in J2EE + Container - managed environment + + The J2EE container model has fallen under fire over the + past few years for many valid reasons. There are some things that + the J2EE container do very well though that projects should + consider when planning for scalability with batch architectures. + Commercial and open source containers like WebSphere, BEA and + JBOSS typically: + + + + + + manage datasources effectively along with attendent + services like prepared statement caching. + + + + + + manage transactions effectively including many + configurable properties for long lived transactions. + + + + + + manage thread pools more effectively. + + + + + + supply robust implementations of JTA, a requirement + when batch jobs output to multiple XA resources like JMS and + JDBC. + + + + + + manage distribution effectively including domains, + clusters and cells + + + + + + provide robust JMX management for configuring, managing + and administering distributed applications. + + + + + + workload management facilities (clusters) provided by + J2EE vendors + + + + + + Projects are encouraged to deploy batch applications with + the simplest configuration possible, but when federated JVMs are + a requirement to process volumes of data within a batch window, + batch-in-container provides an effective way of distributing the + processing. Spring Batch supports this through a simple change in + configuration. [Work in progress on the exact implementation - + being released as part of M2]. + +
+
+ + + diff --git a/docs/src/site/docbook/reference/common-patterns.xml b/docs/src/site/docbook/reference/common-patterns.xml index f2e61800f..b834dfbc7 100644 --- a/docs/src/site/docbook/reference/common-patterns.xml +++ b/docs/src/site/docbook/reference/common-patterns.xml @@ -311,7 +311,7 @@ @@ -331,7 +331,7 @@ diff --git a/docs/src/site/docbook/reference/container-overview.xml b/docs/src/site/docbook/reference/container-overview.xml index 9076b5ef1..193ba2768 100644 --- a/docs/src/site/docbook/reference/container-overview.xml +++ b/docs/src/site/docbook/reference/container-overview.xml @@ -39,7 +39,7 @@ diff --git a/docs/src/site/docbook/reference/domain.xml b/docs/src/site/docbook/reference/domain.xml index 2a625148e..b57cbd98a 100644 --- a/docs/src/site/docbook/reference/domain.xml +++ b/docs/src/site/docbook/reference/domain.xml @@ -46,7 +46,7 @@ @@ -83,7 +83,7 @@ + fileref="images/job-heirarchy.png" /> @@ -178,7 +178,7 @@ + fileref="images/job-heirarchy.png" /> @@ -559,7 +559,7 @@ + fileref="images/jobHeirarchyWithSteps.png" /> diff --git a/docs/src/site/docbook/reference/execution.xml b/docs/src/site/docbook/reference/execution.xml index bd11d6102..d2fa94978 100644 --- a/docs/src/site/docbook/reference/execution.xml +++ b/docs/src/site/docbook/reference/execution.xml @@ -18,7 +18,7 @@ @@ -93,7 +93,7 @@ + fileref="images/run-tier.png" /> @@ -273,7 +273,7 @@ + fileref="images/jobTier.png" /> @@ -313,7 +313,7 @@ @@ -336,7 +336,7 @@ @@ -790,7 +790,7 @@ + fileref="images/application-tier.png" /> diff --git a/docs/src/site/docbook/reference/job.xml b/docs/src/site/docbook/reference/job.xml index e1fc41fcf..10ff2cf5b 100644 --- a/docs/src/site/docbook/reference/job.xml +++ b/docs/src/site/docbook/reference/job.xml @@ -15,7 +15,7 @@ @@ -352,7 +352,7 @@ @@ -374,7 +374,7 @@ @@ -596,7 +596,7 @@ @@ -619,7 +619,7 @@ diff --git a/docs/src/site/docbook/reference/readersAndWriters.xml b/docs/src/site/docbook/reference/readersAndWriters.xml index f77b298e5..1f8f87aa6 100644 --- a/docs/src/site/docbook/reference/readersAndWriters.xml +++ b/docs/src/site/docbook/reference/readersAndWriters.xml @@ -1521,7 +1521,7 @@ @@ -1548,7 +1548,7 @@ @@ -1863,7 +1863,7 @@ @@ -2355,7 +2355,7 @@ + fileref="images/errorOnFlush.png" /> If items are buffered before being written out, any errors encountered will not be thrown until the buffer is flushed just @@ -2381,7 +2381,7 @@ + fileref="images/errorOnWrite.jpg" /> diff --git a/docs/src/site/docbook/reference/schema-appendix.xml b/docs/src/site/docbook/reference/schema-appendix.xml index 9dbe2a36c..cc6b0ae1b 100644 --- a/docs/src/site/docbook/reference/schema-appendix.xml +++ b/docs/src/site/docbook/reference/schema-appendix.xml @@ -26,14 +26,14 @@ their relationships to one another: - + diff --git a/docs/src/site/docbook/reference/spring-batch-intro.xml b/docs/src/site/docbook/reference/spring-batch-intro.xml index 3628a8d6c..b67b3f70a 100644 --- a/docs/src/site/docbook/reference/spring-batch-intro.xml +++ b/docs/src/site/docbook/reference/spring-batch-intro.xml @@ -173,9 +173,9 @@ architecture that supports the extensibility and ease of use for end-user developers. - + @@ -201,4 +201,4 @@ ItemWriter) and the core framework itself. (retry) - \ No newline at end of file + diff --git a/docs/src/site/docbook/reference/step.xml b/docs/src/site/docbook/reference/step.xml index 2f297b0f4..3b2e766e6 100644 --- a/docs/src/site/docbook/reference/step.xml +++ b/docs/src/site/docbook/reference/step.xml @@ -24,7 +24,7 @@ @@ -50,7 +50,7 @@ @@ -988,7 +988,7 @@ @@ -1046,7 +1046,7 @@ diff --git a/docs/src/site/docbook/reference/whatsnew.xml b/docs/src/site/docbook/reference/whatsnew.xml index 66d751f6a..82112a0d0 100644 --- a/docs/src/site/docbook/reference/whatsnew.xml +++ b/docs/src/site/docbook/reference/whatsnew.xml @@ -89,7 +89,7 @@ @@ -163,7 +163,7 @@ @@ -237,7 +237,7 @@ @@ -254,7 +254,7 @@ @@ -277,7 +277,7 @@ @@ -341,7 +341,7 @@ @@ -367,7 +367,7 @@ @@ -382,7 +382,7 @@