Re-organise parallel job to run quicker (not football job any more).

This commit is contained in:
dsyer
2007-12-20 08:55:18 +00:00
parent 6ad57bcb69
commit a29f0f83e8
7 changed files with 242 additions and 306 deletions

View File

@@ -26,12 +26,11 @@ import org.springframework.transaction.support.TransactionSynchronizationAdapter
import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.util.Assert;
public class StagingItemReader extends JdbcDaoSupport implements
ItemReader, ResourceLifecycle, DisposableBean,
public class StagingItemReader extends JdbcDaoSupport implements ItemReader, ResourceLifecycle, DisposableBean,
StepContextAware {
// Key for buffer in transaction synchronization manager
private static final String BUFFER_KEY = StagingItemReader.class.getName()+".BUFFER";
private static final String BUFFER_KEY = StagingItemReader.class.getName() + ".BUFFER";
private static Log logger = LogFactory.getLog(StagingItemReader.class);
@@ -41,7 +40,7 @@ public class StagingItemReader extends JdbcDaoSupport implements
private Object lock = new Object();
private boolean initialized = false;
private volatile boolean initialized = false;
private volatile Iterator keys;
@@ -72,13 +71,14 @@ public class StagingItemReader extends JdbcDaoSupport implements
* @see org.springframework.batch.io.driving.DrivingQueryItemReader#open()
*/
public void open() {
Assert.state(keys == null || initialized,
"Cannot open an already open StagingItemProvider"
+ ", call close() first.");
Assert.state(keys == null || initialized, "Cannot open an already open StagingItemProvider"
+ ", call close() first.");
synchronized (lock) {
keys = retrieveKeys().iterator();
if (keys == null) {
keys = retrieveKeys().iterator();
logger.info("Keys obtained for staging.");
}
}
logger.info("keys: " + keys);
registerSynchronization();
initialized = true;
}
@@ -86,8 +86,7 @@ public class StagingItemReader extends JdbcDaoSupport implements
/**
* Callback for injection of the step context.
*
* @param stepContext
* the stepContext to set
* @param stepContext the stepContext to set
*/
public void setStepContext(StepContext stepContext) {
this.stepContext = stepContext;
@@ -97,24 +96,19 @@ public class StagingItemReader extends JdbcDaoSupport implements
synchronized (lock) {
return getJdbcTemplate()
.query(
return getJdbcTemplate().query(
"SELECT ID FROM BATCH_STAGING WHERE JOB_ID=? AND PROCESSED=? ORDER BY ID",
"SELECT ID FROM BATCH_STAGING WHERE JOB_ID=? AND PROCESSED=? ORDER BY ID",
new Object[] {
stepContext.getStepExecution()
.getJobExecution().getJobId(),
StagingItemProcessor.NEW },
new Object[] { stepContext.getStepExecution().getJobExecution().getJobId(), StagingItemProcessor.NEW },
new RowMapper() {
public Object mapRow(ResultSet rs, int rowNum)
throws SQLException {
return new Long(rs.getLong(1));
}
}
new RowMapper() {
public Object mapRow(ResultSet rs, int rowNum) throws SQLException {
return new Long(rs.getLong(1));
}
}
);
);
}
@@ -130,26 +124,19 @@ public class StagingItemReader extends JdbcDaoSupport implements
if (id == null) {
return null;
}
Object result = getJdbcTemplate().queryForObject(
"SELECT VALUE FROM BATCH_STAGING WHERE ID=?",
Object result = getJdbcTemplate().queryForObject("SELECT VALUE FROM BATCH_STAGING WHERE ID=?",
new Object[] { id }, new RowMapper() {
public Object mapRow(ResultSet rs, int rowNum)
throws SQLException {
public Object mapRow(ResultSet rs, int rowNum) throws SQLException {
byte[] blob = lobHandler.getBlobAsBytes(rs, 1);
return SerializationUtils.deserialize(blob);
}
});
// Update now - changes will rollback if there is a problem later.
int count = getJdbcTemplate()
.update(
"UPDATE BATCH_STAGING SET PROCESSED=? WHERE ID=? AND PROCESSED=?",
new Object[] { StagingItemProcessor.DONE, id,
StagingItemProcessor.NEW });
int count = getJdbcTemplate().update("UPDATE BATCH_STAGING SET PROCESSED=? WHERE ID=? AND PROCESSED=?",
new Object[] { StagingItemProcessor.DONE, id, StagingItemProcessor.NEW });
if (count != 1) {
throw new OptimisticLockingFailureException(
"The staging record with ID="
+ id
+ " was updated concurrently when trying to mark as complete (updated "+count+" records.");
throw new OptimisticLockingFailureException("The staging record with ID=" + id
+ " was updated concurrently when trying to mark as complete (updated " + count + " records.");
}
return result;
}
@@ -169,7 +156,8 @@ public class StagingItemReader extends JdbcDaoSupport implements
logger.debug("Retrieved key from list: " + key);
}
}
} else {
}
else {
logger.debug("Retrieved key from buffer: " + key);
}
return key;
@@ -178,11 +166,9 @@ public class StagingItemReader extends JdbcDaoSupport implements
private StagingBuffer getBuffer() {
if (!TransactionSynchronizationManager.hasResource(BUFFER_KEY)) {
TransactionSynchronizationManager.bindResource(BUFFER_KEY,
new StagingBuffer());
TransactionSynchronizationManager.bindResource(BUFFER_KEY, new StagingBuffer());
}
return (StagingBuffer) TransactionSynchronizationManager
.getResource(BUFFER_KEY);
return (StagingBuffer) TransactionSynchronizationManager.getResource(BUFFER_KEY);
}
public boolean recover(Object data, Throwable cause) {
@@ -196,8 +182,7 @@ public class StagingItemReader extends JdbcDaoSupport implements
* initialized.
*/
protected void registerSynchronization() {
BatchTransactionSynchronizationManager
.registerSynchronization(synchronization);
BatchTransactionSynchronizationManager.registerSynchronization(synchronization);
}
/*
@@ -221,12 +206,12 @@ public class StagingItemReader extends JdbcDaoSupport implements
/**
* Encapsulates transaction events handling.
*/
private class StagingInputTransactionSynchronization extends
TransactionSynchronizationAdapter {
private class StagingInputTransactionSynchronization extends TransactionSynchronizationAdapter {
public void afterCompletion(int status) {
if (status == TransactionSynchronization.STATUS_ROLLED_BACK) {
transactionRolledBack();
} else if (status == TransactionSynchronization.STATUS_COMMITTED) {
}
else if (status == TransactionSynchronization.STATUS_COMMITTED) {
transactionCommitted();
}
}
@@ -235,6 +220,7 @@ public class StagingItemReader extends JdbcDaoSupport implements
private class StagingBuffer {
private List list = new ArrayList();
private Iterator iter = new ArrayList().iterator();
public Long next() {

View File

@@ -76,8 +76,6 @@
</property>
</bean>
<bean id="fileLocator" class="org.springframework.core.io.ClassPathResource">
<constructor-arg type="java.lang.String"
value="data/fixedLengthImportJob/input/20070122.teststream.ImportTradeDataStep.txt" />
@@ -87,7 +85,4 @@
<bean parent="customEditorConfigurer"/>
<!-- register the step scope with the application context -->
<bean class="org.springframework.batch.execution.scope.StepScope" />
</beans>

View File

@@ -1,9 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:aop="http://www.springframework.org/schema/aop"
xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:p="http://www.springframework.org/schema/p"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:aop="http://www.springframework.org/schema/aop"
xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:p="http://www.springframework.org/schema/p"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
@@ -12,209 +12,161 @@
<bean parent="stepScope" />
<bean parent="jobConfigurationRegistryBeanPostProcessor" />
<bean id="parallelJob"
class="org.springframework.batch.core.configuration.JobConfiguration">
<property name="restartable" value="true" />
<property name="startLimit" value="100" />
<bean id="fixedLengthImportJob" parent="simpleJob">
<property name="steps">
<list>
<bean id="staging" parent="simpleStep">
<property name="tasklet">
<bean
class="org.springframework.batch.execution.tasklet.RestartableItemOrientedTasklet">
<property name="itemReader" ref="playerFileItemReader" />
<property name="itemProcessor">
<bean
class="org.springframework.batch.sample.item.processor.StagingItemProcessor"
scope="step">
<aop:scoped-proxy />
<property name="dataSource"
ref="dataSource" />
<property name="incrementer">
<bean
parent="incrementerParent">
<property
name="incrementerName" value="BATCH_STAGING_SEQ" />
</bean>
</property>
</bean>
</property>
</bean>
</property>
<property name="commitInterval" value="10000"></property>
<property name="startLimit" value="100" />
<property name="saveRestartData" value="true" />
<property name="allowStartIfComplete" value="false" />
</bean>
<bean id="playerload"
class="org.springframework.batch.execution.step.RepeatOperationsStepConfiguration">
<property name="tasklet">
<bean
class="org.springframework.batch.execution.tasklet.ItemOrientedTasklet">
<property name="itemReader">
<bean
class="org.springframework.batch.sample.item.provider.StagingItemReader"
scope="step">
<aop:scoped-proxy />
<property name="dataSource"
ref="dataSource" />
</bean>
</property>
<property name="itemProcessor">
<bean
class="org.springframework.batch.sample.item.processor.PlayerItemProcessor">
<property name="playerDao">
<bean
class="org.springframework.batch.sample.dao.JdbcPlayerDao">
<property name="dataSource"
ref="dataSource" />
</bean>
</property>
</bean>
</property>
</bean>
</property>
<property name="chunkOperations">
<bean
class="org.springframework.batch.repeat.support.RepeatTemplate">
<property name="completionPolicy">
<bean
class="org.springframework.batch.repeat.policy.SimpleCompletionPolicy">
<property name="chunkSize"
value="10" />
</bean>
</property>
</bean>
</property>
<property name="stepOperations">
<bean
class="org.springframework.batch.repeat.support.TaskExecutorRepeatTemplate">
<property name="taskExecutor">
<bean
class="org.springframework.core.task.SimpleAsyncTaskExecutor" />
</property>
</bean>
</property>
</bean>
<bean id="gameLoad"
class="org.springframework.batch.execution.step.SimpleStepConfiguration">
<property name="commitInterval" value="1000" />
<property name="startLimit" value="100" />
<property name="saveRestartData" value="true" />
<property name="tasklet">
<bean
class="org.springframework.batch.execution.tasklet.RestartableItemOrientedTasklet">
<property name="itemReader" ref="gameFileItemReader" />
<property name="itemProcessor">
<bean
class="org.springframework.batch.item.processor.ItemWriterItemProcessor">
<property name="itemWriter">
<bean
class="org.springframework.batch.sample.dao.JdbcGameDao">
<property name="dataSource"
ref="dataSource" />
</bean>
</property>
</bean>
</property>
</bean>
</property>
</bean>
<bean id="playerSummarization"
class="org.springframework.batch.execution.step.SimpleStepConfiguration">
<property name="commitInterval" value="100" />
<property name="startLimit" value="100" />
<property name="saveRestartData" value="true" />
<property name="tasklet">
<bean
class="org.springframework.batch.execution.tasklet.RestartableItemOrientedTasklet">
<property name="itemReader" ref="playerSummarizationSource" />
<property name="itemProcessor">
<bean
class="org.springframework.batch.item.processor.ItemWriterItemProcessor">
<property name="itemWriter">
<bean
class="org.springframework.batch.sample.dao.JdbcPlayerSummaryDao">
<property name="dataSource"
ref="dataSource" />
</bean>
</property>
</bean>
</property>
</bean>
</property>
</bean>
</list>
</property>
</bean>
<bean id="playerFileItemReader"
class="org.springframework.batch.io.file.support.DefaultFlatFileItemReader"
scope="step">
<aop:scoped-proxy />
<property name="resource"
value="classpath:data/footballjob/input/player.csv" />
<property name="tokenizer">
<bean
class="org.springframework.batch.io.file.support.transform.DelimitedLineTokenizer">
<property name="names"
value="ID,lastName,firstName,position,birthYear,debutYear" />
</bean>
</property>
<property name="fieldSetMapper">
<bean
class="org.springframework.batch.sample.mapping.PlayerMapper" />
</property>
</bean>
<bean id="gameFileItemReader"
class="org.springframework.batch.io.file.support.DefaultFlatFileItemReader"
scope="step">
<aop:scoped-proxy />
<property name="resource"
value="classpath:data/footballjob/input/games.csv" />
<property name="tokenizer">
<bean
class="org.springframework.batch.io.file.support.transform.DelimitedLineTokenizer">
<property name="names"
value="id,year,team,week,opponent,completes,attempts,passingYards,passingTd,interceptions,rushes,rushYards,receptions,receptionYards,totalTd" />
</bean>
</property>
<property name="fieldSetMapper">
<bean
class="org.springframework.batch.sample.mapping.GameMapper" />
</property>
</bean>
<bean id="playerSummarizationSource"
class="org.springframework.batch.io.cursor.JdbcCursorItemReader"
scope="step">
<aop:scoped-proxy />
<property name="dataSource" ref="dataSource" />
<property name="mapper">
<bean
class="org.springframework.batch.sample.mapping.PlayerSummaryMapper" />
</property>
<property name="sql">
<value>
SELECT games.player_id, games.year_no, SUM(COMPLETES),
SUM(ATTEMPTS), SUM(PASSING_YARDS), SUM(PASSING_TD),
SUM(INTERCEPTIONS), SUM(RUSHES), SUM(RUSH_YARDS),
SUM(RECEPTIONS), SUM(RECEPTIONS_YARDS), SUM(TOTAL_TD)
from games, players where players.player_id =
games.player_id group by games.player_id, games.year_no
</value>
</property>
</bean>
<aop:config>
<aop:aspect id="moduleLogging" ref="itemProcessorLogAdvice">
<aop:after
pointcut="execution( * org.springframework.batch.item.ItemProcessor+.process(Object)) and args(item)"
method="doStronglyTypedLogging" />
</aop:aspect>
</aop:config>
<bean
class="org.springframework.batch.execution.tasklet.RestartableItemOrientedTasklet">
<property name="itemReader">
<bean
class="org.springframework.batch.item.provider.ValidatingItemReader">
<property name="itemReader"
ref="fileInputTemplate" />
<property name="validator"
ref="fixedValidator" />
</bean>
</property>
<property name="itemProcessor">
<bean
class="org.springframework.batch.sample.item.processor.StagingItemProcessor"
scope="step">
<aop:scoped-proxy />
<property name="dataSource"
ref="dataSource" />
<property name="incrementer">
<bean
parent="incrementerParent">
<property
name="incrementerName" value="BATCH_STAGING_SEQ" />
</bean>
</property>
</bean>
</property>
</bean>
</property>
<property name="commitInterval" value="2"/>
<property name="startLimit" value="100" />
<property name="saveRestartData" value="true" />
<property name="allowStartIfComplete" value="false" />
</bean>
<bean id="loading"
class="org.springframework.batch.execution.step.RepeatOperationsStepConfiguration">
<property name="tasklet">
<bean
class="org.springframework.batch.execution.tasklet.ItemOrientedTasklet">
<property name="itemReader">
<bean
class="org.springframework.batch.sample.item.provider.StagingItemReader"
scope="step">
<aop:scoped-proxy />
<property name="dataSource"
ref="dataSource" />
</bean>
</property>
<property name="itemProcessor">
<bean
class="org.springframework.batch.sample.item.processor.TradeProcessor">
<property name="writer"
ref="tradeDao" />
</bean>
</property>
</bean>
</property>
<property name="chunkOperations">
<bean
class="org.springframework.batch.repeat.support.RepeatTemplate">
<property name="completionPolicy">
<bean
class="org.springframework.batch.repeat.policy.SimpleCompletionPolicy">
<property name="chunkSize"
value="2" />
</bean>
</property>
</bean>
</property>
<property name="stepOperations">
<bean
class="org.springframework.batch.repeat.support.TaskExecutorRepeatTemplate">
<property name="taskExecutor">
<bean
class="org.springframework.core.task.SimpleAsyncTaskExecutor" />
</property>
</bean>
</property>
</bean>
</list>
</property>
</bean>
<!-- INFRASTRUCTURE SETUP -->
<!-- This input source is injected into the test case to verify the output - not used by the job at all -->
<bean id="testInputTemplate"
class="org.springframework.batch.io.file.support.DefaultFlatFileItemReader">
<property name="resource" ref="fileLocator" />
<property name="tokenizer" ref="fixedFileDescriptor" />
<property name="fieldSetMapper" ref="fieldSetMapper" />
</bean>
<bean id="fileInputTemplate" parent="testInputTemplate"
autowire-candidate="false" scope="step">
<aop:scoped-proxy />
</bean>
<bean id="fixedFileDescriptor"
class="org.springframework.batch.io.file.support.transform.FixedLengthTokenizer">
<property name="names" value="ISIN, Quantity, Price, Customer" />
<property name="columns" value="1-12, 13-15, 16-20, 21-29" />
</bean>
<bean id="fixedValidator"
class="org.springframework.batch.item.validator.SpringValidator">
<property name="validator">
<bean id="tradeValidator"
class="org.springmodules.validation.valang.ValangValidator">
<property name="valang">
<value>
<![CDATA[
{ isin : length(?) < 13 : 'ISIN too long' : 'isin_length' : 12}
]]>
</value>
</property>
</bean>
</property>
</bean>
<bean id="tradeDao"
class="org.springframework.batch.sample.dao.JdbcTradeWriter">
<property name="jdbcTemplate" ref="jdbcTemplate" />
<property name="incrementer">
<bean parent="incrementerParent">
<property name="incrementerName" value="TRADE_SEQ" />
</bean>
</property>
</bean>
<bean id="fileLocator"
class="org.springframework.core.io.ClassPathResource">
<constructor-arg type="java.lang.String"
value="data/fixedLengthImportJob/input/20070122.teststream.ImportTradeDataStep.txt" />
</bean>
<bean id="fieldSetMapper"
class="org.springframework.batch.sample.mapping.TradeFieldSetMapper" />
<bean parent="customEditorConfigurer" />
<aop:config>
<aop:aspect id="moduleLogging" ref="itemProcessorLogAdvice">
<aop:after
pointcut="execution( * org.springframework.batch.item.ItemProcessor+.process(Object)) and args(item)"
method="doStronglyTypedLogging" />
</aop:aspect>
</aop:config>
</beans>

View File

@@ -2,11 +2,20 @@
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n
log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %t %c{1}:%L - %m%n
log4j.appender.chainsaw=org.apache.log4j.RollingFileAppender
log4j.appender.chainsaw.File=out.xml
log4j.appender.chainsaw.Append=false
log4j.appender.chainsaw.Threshold=debug
log4j.appender.chainsaw.MaxFileSize=10MB
log4j.appender.chainsaw.MaxBackupIndex=2
log4j.appender.chainsaw.layout=org.apache.log4j.xml.XMLLayout
### set log levels - for more verbose logging change 'info' to 'debug' ###
log4j.rootLogger=info, stdout
# log4j.rootLogger=info, stdout, chainsaw
### enable the following line if you want to track down connection ###
### leakages when using DriverManagerConnectionProvider ###

View File

@@ -2,7 +2,6 @@ package org.springframework.batch.sample;
import javax.sql.DataSource;
import org.springframework.batch.sample.item.processor.StagingItemProcessor;
import org.springframework.jdbc.core.JdbcOperations;
import org.springframework.jdbc.core.JdbcTemplate;
@@ -16,21 +15,12 @@ public class FootballJobFunctionalTests extends
}
protected String[] getConfigLocations() {
return new String[] { "jobs/parallelJob.xml##" };
return new String[] { "jobs/footballJob.xml##" };
}
protected void validatePostConditions() throws Exception {
int count;
count = jdbcTemplate.queryForInt(
"SELECT COUNT(*) from BATCH_STAGING where PROCESSED=?",
new Object[] {StagingItemProcessor.NEW});
assertEquals(0, count);
int total = jdbcTemplate.queryForInt(
"SELECT COUNT(*) from BATCH_STAGING");
count = jdbcTemplate.queryForInt(
"SELECT COUNT(*) from BATCH_STAGING where PROCESSED=?",
new Object[] {StagingItemProcessor.DONE});
assertEquals(total, count);
int count = jdbcTemplate.queryForInt("SELECT COUNT(*) from PLAYER_SUMMARY");
assertTrue(count>0);
}
}

View File

@@ -0,0 +1,36 @@
package org.springframework.batch.sample;
import javax.sql.DataSource;
import org.springframework.batch.sample.item.processor.StagingItemProcessor;
import org.springframework.jdbc.core.JdbcOperations;
import org.springframework.jdbc.core.JdbcTemplate;
public class ParallelJobFunctionalTests extends
AbstractValidatingBatchLauncherTests {
private JdbcOperations jdbcTemplate;
public void setDataSource(DataSource dataSource) {
this.jdbcTemplate = new JdbcTemplate(dataSource);
}
protected String[] getConfigLocations() {
return new String[] { "jobs/parallelJob.xml" };
}
protected void validatePostConditions() throws Exception {
int count;
count = jdbcTemplate.queryForInt(
"SELECT COUNT(*) from BATCH_STAGING where PROCESSED=?",
new Object[] {StagingItemProcessor.NEW});
assertEquals(0, count);
int total = jdbcTemplate.queryForInt(
"SELECT COUNT(*) from BATCH_STAGING");
count = jdbcTemplate.queryForInt(
"SELECT COUNT(*) from BATCH_STAGING where PROCESSED=?",
new Object[] {StagingItemProcessor.DONE});
assertEquals(total, count);
}
}

View File

@@ -1,32 +0,0 @@
### direct log messages to stdout ###
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n
log4j.appender.chainsaw=org.apache.log4j.RollingFileAppender
log4j.appender.chainsaw.File=out.xml
log4j.appender.chainsaw.Append=false
log4j.appender.chainsaw.Threshold=debug
log4j.appender.chainsaw.MaxFileSize=10MB
log4j.appender.chainsaw.MaxBackupIndex=2
log4j.appender.chainsaw.layout=org.apache.log4j.xml.XMLLayout
### set log levels - for more verbose logging change 'info' to 'debug' ###
log4j.rootLogger=info, stdout
# log4j.rootLogger=info, stdout, chainsaw
### enable the following line if you want to track down connection ###
### leakages when using DriverManagerConnectionProvider ###
#log4j.logger.org.hibernate.connection.DriverManagerConnectionProvider=trace
### enable spring
log4j.logger.org.springframework=error
log4j.logger.org.springframework.batch.sample=info
### debug your specific package or classes with the following example
log4j.logger.org.springframework.batch.sample.module.OrderDataProvider=debug
log4j.logger.org.springframework.batch.container.common.module.process.support.DefaultXmlDataProvider=debug
#log4j.logger.org.springframework.jdbc.core=debug
#log4j.logger.org.springframework.transaction=debug