BATCH-1908. Added batch inserts to JdbcExecutionContextDao and JdbcStepExecutionDao. Added JobRepository.addAll(Collection<StepExecution>)

This commit is contained in:
David Turanski
2013-03-05 18:11:43 -05:00
committed by Michael Minella
parent feadfd35ca
commit 2f9df44443
12 changed files with 248 additions and 45 deletions

View File

@@ -187,11 +187,11 @@ public class SimpleStepExecutionSplitter implements StepExecutionSplitter, Initi
boolean startable = getStartable(currentStepExecution, context.getValue());
if (startable) {
jobRepository.add(currentStepExecution);
set.add(currentStepExecution);
}
}
jobRepository.addAll(set);
return set;

View File

@@ -16,6 +16,8 @@
package org.springframework.batch.core.repository;
import java.util.Collection;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobInstance;
@@ -39,6 +41,7 @@ import org.springframework.transaction.annotation.Isolation;
* @author Lucas Ward
* @author Dave Syer
* @author Robert Kasanicky
* @author David Turanski
*/
public interface JobRepository {
@@ -115,6 +118,17 @@ public interface JobRepository {
*/
void add(StepExecution stepExecution);
/**
* Save a collection of {@link StepExecution}s and each {@link ExecutionContext}. The
* StepExecution ID will be assigned - it is not permitted that an ID be assigned before calling
* this method. Instead, it should be left blank, to be assigned by {@link JobRepository}.
*
* Preconditions: {@link StepExecution} must have a valid {@link Step}.
*
* @param stepExecution
*/
void addAll(Collection<StepExecution> stepExecutions);
/**
* Update the {@link StepExecution} (but not its {@link ExecutionContext}).
*

View File

@@ -16,6 +16,8 @@
package org.springframework.batch.core.repository.dao;
import java.util.Collection;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.item.ExecutionContext;
@@ -24,6 +26,7 @@ import org.springframework.batch.item.ExecutionContext;
* DAO interface for persisting and retrieving {@link ExecutionContext}s.
*
* @author Robert Kasanicky
* @author David Turanski
*/
public interface ExecutionContextDao {
@@ -52,6 +55,13 @@ public interface ExecutionContextDao {
* @param stepExecution
*/
void saveExecutionContext(final StepExecution stepExecution);
/**
* Persist the execution context associated with each stepExecution in a given collection,
* persistent entry for the context should not exist yet.
* @param stepExecution
*/
void saveExecutionContexts(final Collection<StepExecution> stepExecutions);
/**
* Persist the updates of execution context associated with the given

View File

@@ -22,7 +22,9 @@ import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -32,6 +34,7 @@ import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.repository.ExecutionContextSerializer;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.core.serializer.Serializer;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.PreparedStatementSetter;
import org.springframework.jdbc.core.simple.ParameterizedRowMapper;
import org.springframework.jdbc.support.lob.DefaultLobHandler;
@@ -48,6 +51,7 @@ import org.springframework.util.Assert;
* @author Robert Kasanicky
* @author Thomas Risberg
* @author Michael Minella
* @author David Turanski
*/
public class JdbcExecutionContextDao extends AbstractJdbcBatchMetadataDao implements ExecutionContextDao {
@@ -109,8 +113,7 @@ public class JdbcExecutionContextDao extends AbstractJdbcBatchMetadataDao implem
new ExecutionContextRowMapper(), executionId);
if (results.size() > 0) {
return results.get(0);
}
else {
} else {
return new ExecutionContext();
}
}
@@ -124,8 +127,7 @@ public class JdbcExecutionContextDao extends AbstractJdbcBatchMetadataDao implem
new ExecutionContextRowMapper(), executionId);
if (results.size() > 0) {
return results.get(0);
}
else {
} else {
return new ExecutionContext();
}
}
@@ -180,6 +182,20 @@ public class JdbcExecutionContextDao extends AbstractJdbcBatchMetadataDao implem
persistSerializedContext(executionId, serializedContext, INSERT_STEP_EXECUTION_CONTEXT);
}
@Override
public void saveExecutionContexts(Collection<StepExecution> stepExecutions) {
Assert.notNull(stepExecutions, "Attempt to save an null collection of step executions");
Map<Long, String> serializedContexts = new HashMap<Long, String>(stepExecutions.size());
for (StepExecution stepExecution : stepExecutions) {
Long executionId = stepExecution.getId();
ExecutionContext executionContext = stepExecution.getExecutionContext();
Assert.notNull(executionId, "ExecutionId must not be null.");
Assert.notNull(executionContext, "The ExecutionContext must not be null.");
serializedContexts.put(executionId, serializeContext(executionContext));
}
persistSerializedContexts(serializedContexts, INSERT_STEP_EXECUTION_CONTEXT);
}
public void setLobHandler(LobHandler lobHandler) {
this.lobHandler = lobHandler;
}
@@ -203,8 +219,7 @@ public class JdbcExecutionContextDao extends AbstractJdbcBatchMetadataDao implem
// 2-byte chars
shortContext = serializedContext.substring(0, shortContextLength - 8) + " ...";
longContext = serializedContext;
}
else {
} else {
shortContext = serializedContext;
longContext = null;
}
@@ -215,8 +230,7 @@ public class JdbcExecutionContextDao extends AbstractJdbcBatchMetadataDao implem
ps.setString(1, shortContext);
if (longContext != null) {
lobHandler.getLobCreator().setClobAsString(ps, 2, longContext);
}
else {
} else {
ps.setNull(2, getClobTypeToUse());
}
ps.setLong(3, executionId);
@@ -224,6 +238,47 @@ public class JdbcExecutionContextDao extends AbstractJdbcBatchMetadataDao implem
});
}
/**
* @param executionId
* @param serializedContext
* @param sql with parameters (shortContext, longContext, executionId)
*/
private void persistSerializedContexts(final Map<Long, String> serializedContexts, String sql) {
final Iterator<Long> executionIdIterator = serializedContexts.keySet().iterator();
getJdbcTemplate().batchUpdate(getQuery(sql), new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
Long executionId = executionIdIterator.next();
String serializedContext = serializedContexts.get(executionId);
String shortContext;
String longContext;
if (serializedContext.length() > shortContextLength) {
// Overestimate length of ellipsis to be on the safe side with
// 2-byte chars
shortContext = serializedContext.substring(0, shortContextLength - 8) + " ...";
longContext = serializedContext;
} else {
shortContext = serializedContext;
longContext = null;
}
ps.setString(1, shortContext);
if (longContext != null) {
lobHandler.getLobCreator().setClobAsString(ps, 2, longContext);
} else {
ps.setNull(2, getClobTypeToUse());
}
ps.setLong(3, executionId);
}
@Override
public int getBatchSize() {
return serializedContexts.size();
}
});
}
@SuppressWarnings("unchecked")
private String serializeContext(ExecutionContext ctx) {
Map<String, Object> m = new HashMap<String, Object>();
@@ -237,8 +292,7 @@ public class JdbcExecutionContextDao extends AbstractJdbcBatchMetadataDao implem
try {
serializer.serialize(m, out);
results = new String(out.toByteArray(), "ISO-8859-1");
}
catch (IOException ioe) {
} catch (IOException ioe) {
throw new IllegalArgumentException("Could not serialize the execution context", ioe);
}
@@ -260,8 +314,7 @@ public class JdbcExecutionContextDao extends AbstractJdbcBatchMetadataDao implem
try {
ByteArrayInputStream in = new ByteArrayInputStream(serializedContext.getBytes("ISO-8859-1"));
map = (Map<String, Object>) serializer.deserialize(in);
}
catch (IOException ioe) {
} catch (IOException ioe) {
throw new IllegalArgumentException("Unable to deserialize the execution context", ioe);
}
for (Map.Entry<String, Object> entry : map.entrySet()) {

View File

@@ -16,9 +16,15 @@
package org.springframework.batch.core.repository.dao;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.sql.Types;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -29,6 +35,7 @@ import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.StepExecution;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.dao.OptimisticLockingFailureException;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.simple.ParameterizedRowMapper;
import org.springframework.jdbc.support.incrementer.DataFieldMaxValueIncrementer;
import org.springframework.util.Assert;
@@ -49,6 +56,7 @@ import org.springframework.util.Assert;
* @author Lucas Ward
* @author Dave Syer
* @author Robert Kasanicky
* @author David Turanski
*
* @see StepExecutionDao
*/
@@ -106,32 +114,88 @@ public class JdbcStepExecutionDao extends AbstractJdbcBatchMetadataDao implement
*/
@Override
public void saveStepExecution(StepExecution stepExecution) {
List<Object[]> parameters = buildStepExecutionParameters(stepExecution);
Object[] parameterValues = (Object[])parameters.get(0);
//Template expects an int array fails with Integer
int[] parameterTypes = new int[parameters.get(1).length];
for (int i = 0; i < parameterTypes.length; i++) {
parameterTypes[i] = (Integer)parameters.get(1)[i];
}
getJdbcTemplate().update(getQuery(SAVE_STEP_EXECUTION), parameterValues, parameterTypes);
}
/**
* Batch insert StepExecutions
* @see StepExecutionDao#saveStepExecutions(Collection)
*/
@Override
public void saveStepExecutions(final Collection<StepExecution> stepExecutions) {
Assert.notNull(stepExecutions, "Attempt to save a null collection of step executions");
final Iterator<StepExecution> iterator = stepExecutions.iterator();
getJdbcTemplate().batchUpdate(getQuery(SAVE_STEP_EXECUTION), new BatchPreparedStatementSetter() {
@Override
public int getBatchSize() {
return stepExecutions.size();
}
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
StepExecution stepExecution = iterator.next();
List<Object[]> parameters = buildStepExecutionParameters(stepExecution);
Object[] parameterValues = (Object[]) parameters.get(0);
Integer[] parameterTypes = (Integer[]) parameters.get(1);
for (int indx = 0; indx < parameterValues.length; indx++) {
switch (parameterTypes[indx]) {
case Types.INTEGER:
ps.setInt(indx + 1, (Integer) parameterValues[indx]);
break;
case Types.VARCHAR:
ps.setString(indx + 1, (String) parameterValues[indx]);
break;
case Types.TIMESTAMP:
if (parameterValues[indx] !=null) {
ps.setTimestamp(indx + 1, new Timestamp(((java.util.Date) parameterValues[indx]).getTime()));
}
break;
case Types.BIGINT:
ps.setLong(indx + 1, (Long) parameterValues[indx]);
break;
default:
throw new IllegalArgumentException(
"unsupported SQL parameter type for step execution field index " + i);
}
}
}
});
}
private List<Object[]> buildStepExecutionParameters(StepExecution stepExecution) {
Assert.isNull(stepExecution.getId(),
"to-be-saved (not updated) StepExecution can't already have an id assigned");
Assert.isNull(stepExecution.getVersion(),
"to-be-saved (not updated) StepExecution can't already have a version assigned");
validateStepExecution(stepExecution);
String exitDescription = truncateExitDescription(stepExecution.getExitStatus().getExitDescription());
stepExecution.setId(stepExecutionIncrementer.nextLongValue());
stepExecution.incrementVersion(); // should be 0 now
Object[] parameters = new Object[] { stepExecution.getId(), stepExecution.getVersion(),
stepExecution.incrementVersion(); //Should be 0
List<Object[]> parameters = new ArrayList<Object[]>();
String exitDescription = truncateExitDescription(stepExecution.getExitStatus().getExitDescription());
Object[] parameterValues = new Object[] { stepExecution.getId(), stepExecution.getVersion(),
stepExecution.getStepName(), stepExecution.getJobExecutionId(), stepExecution.getStartTime(),
stepExecution.getEndTime(), stepExecution.getStatus().toString(), stepExecution.getCommitCount(),
stepExecution.getReadCount(), stepExecution.getFilterCount(), stepExecution.getWriteCount(),
stepExecution.getExitStatus().getExitCode(), exitDescription, stepExecution.getReadSkipCount(),
stepExecution.getWriteSkipCount(), stepExecution.getProcessSkipCount(),
stepExecution.getRollbackCount(), stepExecution.getLastUpdated() };
getJdbcTemplate().update(
getQuery(SAVE_STEP_EXECUTION),
parameters,
new int[] { Types.BIGINT, Types.INTEGER, Types.VARCHAR, Types.BIGINT, Types.TIMESTAMP,
Types.TIMESTAMP, Types.VARCHAR, Types.INTEGER, Types.INTEGER, Types.INTEGER, Types.INTEGER,
Types.VARCHAR, Types.VARCHAR, Types.INTEGER, Types.INTEGER, Types.INTEGER, Types.INTEGER,
Types.TIMESTAMP });
Integer[] parameterTypes = new Integer[] { Types.BIGINT, Types.INTEGER, Types.VARCHAR, Types.BIGINT,
Types.TIMESTAMP, Types.TIMESTAMP, Types.VARCHAR, Types.INTEGER, Types.INTEGER, Types.INTEGER,
Types.INTEGER, Types.VARCHAR, Types.VARCHAR, Types.INTEGER, Types.INTEGER, Types.INTEGER,
Types.INTEGER, Types.TIMESTAMP };
parameters.add(0, Arrays.copyOf(parameterValues,parameterValues.length));
parameters.add(1, Arrays.copyOf(parameterTypes,parameterTypes.length));
return parameters;
}
/**
@@ -171,13 +235,13 @@ public class JdbcStepExecutionDao extends AbstractJdbcBatchMetadataDao implement
stepExecution.getReadSkipCount(), stepExecution.getProcessSkipCount(),
stepExecution.getWriteSkipCount(), stepExecution.getRollbackCount(),
stepExecution.getLastUpdated(), stepExecution.getId(), stepExecution.getVersion() };
int count = getJdbcTemplate().update(
getQuery(UPDATE_STEP_EXECUTION),
parameters,
new int[] { Types.TIMESTAMP, Types.TIMESTAMP, Types.VARCHAR, Types.INTEGER, Types.INTEGER,
Types.INTEGER, Types.INTEGER, Types.VARCHAR, Types.VARCHAR, Types.INTEGER,
Types.INTEGER, Types.INTEGER, Types.INTEGER, Types.INTEGER, Types.TIMESTAMP,
Types.BIGINT, Types.INTEGER });
int count = getJdbcTemplate()
.update(getQuery(UPDATE_STEP_EXECUTION),
parameters,
new int[] { Types.TIMESTAMP, Types.TIMESTAMP, Types.VARCHAR, Types.INTEGER, Types.INTEGER,
Types.INTEGER, Types.INTEGER, Types.VARCHAR, Types.VARCHAR, Types.INTEGER,
Types.INTEGER, Types.INTEGER, Types.INTEGER, Types.INTEGER, Types.TIMESTAMP,
Types.BIGINT, Types.INTEGER });
// Avoid concurrent modifications...
if (count == 0) {
@@ -203,8 +267,7 @@ public class JdbcStepExecutionDao extends AbstractJdbcBatchMetadataDao implement
if (description != null && description.length() > exitMessageLength) {
logger.debug("Truncating long message before update of StepExecution, original message is: " + description);
return description.substring(0, exitMessageLength);
}
else {
} else {
return description;
}
}
@@ -218,8 +281,7 @@ public class JdbcStepExecutionDao extends AbstractJdbcBatchMetadataDao implement
"There can be at most one step execution with given name for single job execution");
if (executions.isEmpty()) {
return null;
}
else {
} else {
return executions.get(0);
}
}

View File

@@ -17,6 +17,7 @@
package org.springframework.batch.core.repository.dao;
import java.io.Serializable;
import java.util.Collection;
import java.util.concurrent.ConcurrentMap;
import org.springframework.batch.core.JobExecution;
@@ -24,12 +25,14 @@ import org.springframework.batch.core.StepExecution;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.support.SerializationUtils;
import org.springframework.batch.support.transaction.TransactionAwareProxyFactory;
import org.springframework.util.Assert;
/**
* In-memory implementation of {@link ExecutionContextDao} backed by maps.
*
* @author Robert Kasanicky
* @author Dave Syer
* @author David Turanski
*/
@SuppressWarnings("serial")
public class MapExecutionContextDao implements ExecutionContextDao {
@@ -145,4 +148,13 @@ public class MapExecutionContextDao implements ExecutionContextDao {
updateExecutionContext(stepExecution);
}
@Override
public void saveExecutionContexts(Collection<StepExecution> stepExecutions) {
Assert.notNull(stepExecutions,"Attempt to save a nulk collection of step executions");
for (StepExecution stepExecution: stepExecutions) {
saveExecutionContext(stepExecution);
}
}
}

View File

@@ -17,6 +17,7 @@ package org.springframework.batch.core.repository.dao;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -137,4 +138,12 @@ public class MapStepExecutionDao implements StepExecutionDao {
}
jobExecution.addStepExecutions(copy);
}
@Override
public void saveStepExecutions(Collection<StepExecution> stepExecutions) {
Assert.notNull(stepExecutions,"Attempt to save an null collect of step executions");
for (StepExecution stepExecution: stepExecutions) {
saveStepExecution(stepExecution);
}
}
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2006-2007 the original author or authors.
* Copyright 2006-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
package org.springframework.batch.core.repository.dao;
import java.util.Collection;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.StepExecution;
@@ -33,6 +34,17 @@ public interface StepExecutionDao {
*/
void saveStepExecution(StepExecution stepExecution);
/**
* Save the given collection of StepExecution as a batch.
*
* Preconditions: StepExecution Id must be null.
*
* Postconditions: StepExecution Id will be set to a unique Long.
*
* @param stepExecutions
*/
void saveStepExecutions(Collection<StepExecution> stepExecutions);
/**
* Update the given StepExecution
*
@@ -50,7 +62,7 @@ public interface StepExecutionDao {
* @return a {@link StepExecution}
*/
StepExecution getStepExecution(JobExecution jobExecution, Long stepExecutionId);
/**
* Retrieve all the {@link StepExecution} for the parent {@link JobExecution}.
*

View File

@@ -17,6 +17,7 @@
package org.springframework.batch.core.repository.support;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.List;
@@ -48,6 +49,7 @@ import org.springframework.util.Assert;
* @author Lucas Ward
* @author Dave Syer
* @author Robert Kasanicky
* @author David Turanski
*
* @see JobRepository
* @see JobInstanceDao
@@ -123,12 +125,11 @@ public class SimpleJobRepository implements JobRepository {
if (status == BatchStatus.COMPLETED || status == BatchStatus.ABANDONED) {
throw new JobInstanceAlreadyCompleteException(
"A job instance already exists and is complete for parameters=" + jobParameters
+ ". If you want to run this job again, change the parameters.");
+ ". If you want to run this job again, change the parameters.");
}
}
executionContext = ecDao.getExecutionContext(jobExecutionDao.getLastJobExecution(jobInstance));
}
else {
} else {
// no job found, create one
jobInstance = jobInstanceDao.createJobInstance(jobName, jobParameters);
executionContext = new ExecutionContext();
@@ -167,6 +168,17 @@ public class SimpleJobRepository implements JobRepository {
ecDao.saveExecutionContext(stepExecution);
}
@Override
public void addAll(Collection<StepExecution> stepExecutions) {
Assert.notNull(stepExecutions, "Attempt to save a null collection of step executions");
for (StepExecution stepExecution : stepExecutions) {
validateStepExecution(stepExecution);
stepExecution.setLastUpdated(new Date(System.currentTimeMillis()));
}
stepExecutionDao.saveStepExecutions(stepExecutions);
ecDao.saveExecutionContexts(stepExecutions);
}
@Override
public void update(StepExecution stepExecution) {
validateStepExecution(stepExecution);
@@ -276,6 +288,4 @@ public class SimpleJobRepository implements JobRepository {
return jobExecution;
}
}

View File

@@ -15,6 +15,8 @@
*/
package org.springframework.batch.core.configuration.xml;
import java.util.Collection;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobInstance;
import org.springframework.batch.core.JobParameters;
@@ -27,6 +29,7 @@ import org.springframework.beans.factory.BeanNameAware;
/**
* @author Dan Garrette
* @author David Turanski
* @since 2.0.1
*/
public class DummyJobRepository implements JobRepository, BeanNameAware {
@@ -88,4 +91,8 @@ public class DummyJobRepository implements JobRepository, BeanNameAware {
public void updateExecutionContext(JobExecution jobExecution) {
}
@Override
public void addAll(Collection<StepExecution> stepExecutions) {
}
}

View File

@@ -15,6 +15,8 @@
*/
package org.springframework.batch.core.step;
import java.util.Collection;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobInstance;
import org.springframework.batch.core.JobParameters;
@@ -23,6 +25,7 @@ import org.springframework.batch.core.repository.JobRepository;
/**
* @author Dave Syer
* @author David Turanski
*
*/
public class JobRepositorySupport implements JobRepository {
@@ -96,4 +99,8 @@ public class JobRepositorySupport implements JobRepository {
public void updateExecutionContext(JobExecution jobExecution) {
}
@Override
public void addAll(Collection<StepExecution> stepExecutions) {
}
}

View File

@@ -11,6 +11,8 @@ import static org.springframework.batch.core.BatchStatus.FAILED;
import static org.springframework.batch.core.BatchStatus.STOPPED;
import static org.springframework.batch.core.BatchStatus.UNKNOWN;
import java.util.Collection;
import org.junit.Before;
import org.junit.Test;
import org.springframework.batch.core.ExitStatus;
@@ -45,6 +47,7 @@ import org.springframework.transaction.support.TransactionSynchronizationManager
*
* @author Lucas Ward
* @author Dave Syer
* @author David Turanski
*
*/
public class TaskletStepExceptionTests {
@@ -467,6 +470,10 @@ public class TaskletStepExceptionTests {
@Override
public void updateExecutionContext(JobExecution jobExecution) {
}
@Override
public void addAll(Collection<StepExecution> stepExecutions) {
}
}
}