Change metrics type from int to long in StepExecution

This change prevents integer overflows in
steps operating on a large number of items.

Resolves #3650
This commit is contained in:
Mahmoud Ben Hassine
2021-01-29 14:35:47 +01:00
parent 78ddf44730
commit 327160a3ab
17 changed files with 96 additions and 87 deletions

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2006-2019 the original author or authors.
* Copyright 2006-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,19 +28,19 @@ import java.io.Serializable;
@SuppressWarnings("serial")
public class StepContribution implements Serializable {
private volatile int readCount = 0;
private volatile long readCount = 0;
private volatile int writeCount = 0;
private volatile long writeCount = 0;
private volatile int filterCount = 0;
private volatile long filterCount = 0;
private final int parentSkipCount;
private final long parentSkipCount;
private volatile int readSkipCount;
private volatile long readSkipCount;
private volatile int writeSkipCount;
private volatile long writeSkipCount;
private volatile int processSkipCount;
private volatile long processSkipCount;
private ExitStatus exitStatus = ExitStatus.EXECUTING;
@@ -76,9 +76,9 @@ public class StepContribution implements Serializable {
/**
* Increment the counter for the number of items processed.
*
* @param count int amount to increment by.
* @param count long amount to increment by.
*/
public void incrementFilterCount(int count) {
public void incrementFilterCount(long count) {
filterCount += count;
}
@@ -92,9 +92,9 @@ public class StepContribution implements Serializable {
/**
* Increment the counter for the number of items written.
*
* @param count int amount to increment by.
* @param count long amount to increment by.
*/
public void incrementWriteCount(int count) {
public void incrementWriteCount(long count) {
writeCount += count;
}
@@ -103,7 +103,7 @@ public class StepContribution implements Serializable {
*
* @return the item counter.
*/
public int getReadCount() {
public long getReadCount() {
return readCount;
}
@@ -112,7 +112,7 @@ public class StepContribution implements Serializable {
*
* @return the item counter.
*/
public int getWriteCount() {
public long getWriteCount() {
return writeCount;
}
@@ -121,7 +121,7 @@ public class StepContribution implements Serializable {
*
* @return the filter counter
*/
public int getFilterCount() {
public long getFilterCount() {
return filterCount;
}
@@ -129,7 +129,7 @@ public class StepContribution implements Serializable {
* @return the sum of skips accumulated in the parent {@link StepExecution}
* and this <code>StepContribution</code>.
*/
public int getStepSkipCount() {
public long getStepSkipCount() {
return readSkipCount + writeSkipCount + processSkipCount + parentSkipCount;
}
@@ -138,7 +138,7 @@ public class StepContribution implements Serializable {
* <code>StepContribution</code> (not including skips accumulated in the
* parent {@link StepExecution}).
*/
public int getSkipCount() {
public long getSkipCount() {
return readSkipCount + writeSkipCount + processSkipCount;
}
@@ -152,9 +152,9 @@ public class StepContribution implements Serializable {
/**
* Increment the read skip count for this contribution
*
* @param count int amount to increment by.
* @param count long amount to increment by.
*/
public void incrementReadSkipCount(int count) {
public void incrementReadSkipCount(long count) {
readSkipCount += count;
}
@@ -175,14 +175,14 @@ public class StepContribution implements Serializable {
/**
* @return the read skip count
*/
public int getReadSkipCount() {
public long getReadSkipCount() {
return readSkipCount;
}
/**
* @return the write skip count
*/
public int getWriteSkipCount() {
public long getWriteSkipCount() {
return writeSkipCount;
}
@@ -191,7 +191,7 @@ public class StepContribution implements Serializable {
*
* @return the process skip count
*/
public int getProcessSkipCount() {
public long getProcessSkipCount() {
return processSkipCount;
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2006-2013 the original author or authors.
* Copyright 2006-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -33,6 +33,7 @@ import org.springframework.util.Assert;
*
* @author Lucas Ward
* @author Dave Syer
* @author Mahmoud Ben Hassine
*
*/
@SuppressWarnings("serial")
@@ -44,19 +45,19 @@ public class StepExecution extends Entity {
private volatile BatchStatus status = BatchStatus.STARTING;
private volatile int readCount = 0;
private volatile long readCount = 0;
private volatile int writeCount = 0;
private volatile long writeCount = 0;
private volatile int commitCount = 0;
private volatile long commitCount = 0;
private volatile int rollbackCount = 0;
private volatile long rollbackCount = 0;
private volatile int readSkipCount = 0;
private volatile long readSkipCount = 0;
private volatile int processSkipCount = 0;
private volatile long processSkipCount = 0;
private volatile int writeSkipCount = 0;
private volatile long writeSkipCount = 0;
private volatile Date startTime = new Date(System.currentTimeMillis());
@@ -70,7 +71,7 @@ public class StepExecution extends Entity {
private volatile boolean terminateOnly;
private volatile int filterCount;
private volatile long filterCount;
private transient volatile List<Throwable> failureExceptions = new CopyOnWriteArrayList<>();
@@ -140,7 +141,7 @@ public class StepExecution extends Entity {
*
* @return the current number of commits
*/
public int getCommitCount() {
public long getCommitCount() {
return commitCount;
}
@@ -149,7 +150,7 @@ public class StepExecution extends Entity {
*
* @param commitCount the current number of commits
*/
public void setCommitCount(int commitCount) {
public void setCommitCount(long commitCount) {
this.commitCount = commitCount;
}
@@ -176,7 +177,7 @@ public class StepExecution extends Entity {
*
* @return the current number of items read for this execution
*/
public int getReadCount() {
public long getReadCount() {
return readCount;
}
@@ -185,7 +186,7 @@ public class StepExecution extends Entity {
*
* @param readCount the current number of read items for this execution
*/
public void setReadCount(int readCount) {
public void setReadCount(long readCount) {
this.readCount = readCount;
}
@@ -194,7 +195,7 @@ public class StepExecution extends Entity {
*
* @return the current number of items written for this execution
*/
public int getWriteCount() {
public long getWriteCount() {
return writeCount;
}
@@ -203,7 +204,7 @@ public class StepExecution extends Entity {
*
* @param writeCount the current number of written items for this execution
*/
public void setWriteCount(int writeCount) {
public void setWriteCount(long writeCount) {
this.writeCount = writeCount;
}
@@ -212,7 +213,7 @@ public class StepExecution extends Entity {
*
* @return the current number of rollbacks for this execution
*/
public int getRollbackCount() {
public long getRollbackCount() {
return rollbackCount;
}
@@ -221,7 +222,7 @@ public class StepExecution extends Entity {
*
* @return the current number of items filtered out of this execution
*/
public int getFilterCount() {
public long getFilterCount() {
return filterCount;
}
@@ -230,15 +231,15 @@ public class StepExecution extends Entity {
* @param filterCount the number of items filtered out of this execution to
* set
*/
public void setFilterCount(int filterCount) {
public void setFilterCount(long filterCount) {
this.filterCount = filterCount;
}
/**
* Setter for number of rollbacks for this execution
* @param rollbackCount int the number of rollbacks.
* @param rollbackCount long the number of rollbacks.
*/
public void setRollbackCount(int rollbackCount) {
public void setRollbackCount(long rollbackCount) {
this.rollbackCount = rollbackCount;
}
@@ -383,7 +384,7 @@ public class StepExecution extends Entity {
/**
* @return the total number of items skipped.
*/
public int getSkipCount() {
public long getSkipCount() {
return readSkipCount + processSkipCount + writeSkipCount;
}
@@ -410,48 +411,48 @@ public class StepExecution extends Entity {
/**
* @return the number of records skipped on read
*/
public int getReadSkipCount() {
public long getReadSkipCount() {
return readSkipCount;
}
/**
* @return the number of records skipped on write
*/
public int getWriteSkipCount() {
public long getWriteSkipCount() {
return writeSkipCount;
}
/**
* Set the number of records skipped on read
*
* @param readSkipCount int containing read skip count to be used for the step execution.
* @param readSkipCount long containing read skip count to be used for the step execution.
*/
public void setReadSkipCount(int readSkipCount) {
public void setReadSkipCount(long readSkipCount) {
this.readSkipCount = readSkipCount;
}
/**
* Set the number of records skipped on write
*
* @param writeSkipCount int containing write skip count to be used for the step execution.
* @param writeSkipCount long containing write skip count to be used for the step execution.
*/
public void setWriteSkipCount(int writeSkipCount) {
public void setWriteSkipCount(long writeSkipCount) {
this.writeSkipCount = writeSkipCount;
}
/**
* @return the number of records skipped during processing
*/
public int getProcessSkipCount() {
public long getProcessSkipCount() {
return processSkipCount;
}
/**
* Set the number of records skipped during processing.
*
* @param processSkipCount int containing process skip count to be used for the step execution.
* @param processSkipCount long containing process skip count to be used for the step execution.
*/
public void setProcessSkipCount(int processSkipCount) {
public void setProcessSkipCount(long processSkipCount) {
this.processSkipCount = processSkipCount;
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2013 the original author or authors.
* Copyright 2013-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -48,6 +48,7 @@ import java.util.List;
*
* @author Michael Minella
* @author Chris Schaefer
* @author Mahmoud Ben Hassine
*
* @param <I> input type for the step
* @param <O> output type for the step
@@ -193,7 +194,7 @@ public class JsrFaultTolerantChunkProcessor<I,O> extends JsrChunkProcessor<I, O>
* @param e the cause of the skip
* @param skipCount the current skip count
*/
private boolean shouldSkip(SkipPolicy policy, Throwable e, int skipCount) {
private boolean shouldSkip(SkipPolicy policy, Throwable e, long skipCount) {
try {
return policy.shouldSkip(e, skipCount);
}

View File

@@ -227,9 +227,9 @@ public class JdbcStepExecutionDao extends AbstractJdbcBatchMetadataDao implement
stepExecution.getWriteSkipCount(), stepExecution.getProcessSkipCount(),
stepExecution.getRollbackCount(), stepExecution.getLastUpdated() };
Integer[] parameterTypes = new Integer[] { Types.BIGINT, Types.INTEGER, Types.VARCHAR, Types.BIGINT,
Types.TIMESTAMP, Types.TIMESTAMP, Types.VARCHAR, Types.INTEGER, Types.INTEGER, Types.INTEGER,
Types.INTEGER, Types.VARCHAR, Types.VARCHAR, Types.INTEGER, Types.INTEGER, Types.INTEGER,
Types.INTEGER, Types.TIMESTAMP };
Types.TIMESTAMP, Types.TIMESTAMP, Types.VARCHAR, Types.BIGINT, Types.BIGINT, Types.BIGINT,
Types.BIGINT, Types.VARCHAR, Types.VARCHAR, Types.BIGINT, Types.BIGINT, Types.BIGINT,
Types.BIGINT, Types.TIMESTAMP };
parameters.add(0, Arrays.copyOf(parameterValues,parameterValues.length));
parameters.add(1, Arrays.copyOf(parameterTypes,parameterTypes.length));
return parameters;

View File

@@ -509,7 +509,7 @@ public class FaultTolerantChunkProcessor<I, O> extends SimpleChunkProcessor<I, O
* @param e the cause of the skip
* @param skipCount the current skip count
*/
private boolean shouldSkip(SkipPolicy policy, Throwable e, int skipCount) {
private boolean shouldSkip(SkipPolicy policy, Throwable e, long skipCount) {
try {
return policy.shouldSkip(e, skipCount);
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2006-2007 the original author or authors.
* Copyright 2006-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -130,7 +130,7 @@ public class FaultTolerantChunkProvider<I> extends SimpleChunkProvider<I> {
* @param e the cause of the skip
* @param skipCount the current skip count
*/
private boolean shouldSkip(SkipPolicy policy, Throwable e, int skipCount) {
private boolean shouldSkip(SkipPolicy policy, Throwable e, long skipCount) {
try {
return policy.shouldSkip(e, skipCount);
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2006-2013 the original author or authors.
* Copyright 2006-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,11 +22,12 @@ package org.springframework.batch.core.step.skip;
*
* @author Ben Hale
* @author Lucas Ward
* @author Mahmoud Ben Hassine
*/
public class AlwaysSkipItemSkipPolicy implements SkipPolicy {
@Override
public boolean shouldSkip(Throwable t, int skipCount) {
public boolean shouldSkip(Throwable t, long skipCount) {
return true;
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2006-2013 the original author or authors.
* Copyright 2006-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@ package org.springframework.batch.core.step.skip;
/**
* @author Dave Syer
* @author Mahmoud Ben Hassine
*
*/
public class CompositeSkipPolicy implements SkipPolicy {
@@ -36,7 +37,7 @@ public class CompositeSkipPolicy implements SkipPolicy {
}
@Override
public boolean shouldSkip(Throwable t, int skipCount) throws SkipLimitExceededException {
public boolean shouldSkip(Throwable t, long skipCount) throws SkipLimitExceededException {
for (SkipPolicy policy : skipPolicies) {
if (policy.shouldSkip(t, skipCount)) {
return true;

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2006-2013 the original author or authors.
* Copyright 2006-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -25,6 +25,7 @@ import org.springframework.classify.SubclassClassifier;
* decision, and then delegates to the classifier result.
*
* @author Dave Syer
* @author Mahmoud Ben Hassine
*
* @see SubclassClassifier
*/
@@ -66,7 +67,7 @@ public class ExceptionClassifierSkipPolicy implements SkipPolicy {
* @throws SkipLimitExceededException if a limit is exceeded
*/
@Override
public boolean shouldSkip(Throwable t, int skipCount) throws SkipLimitExceededException {
public boolean shouldSkip(Throwable t, long skipCount) throws SkipLimitExceededException {
return classifier.classify(t).shouldSkip(t, skipCount);
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2006-2013 the original author or authors.
* Copyright 2006-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -49,10 +49,11 @@ import org.springframework.classify.Classifier;
* @author Robert Kasanicky
* @author Dave Syer
* @author Dan Garrette
* @author Mahmoud Ben Hassine
*/
public class LimitCheckingItemSkipPolicy implements SkipPolicy {
private int skipLimit;
private long skipLimit;
private Classifier<Throwable, Boolean> skippableExceptionClassifier;
@@ -90,7 +91,7 @@ public class LimitCheckingItemSkipPolicy implements SkipPolicy {
*
* @param skipLimit the skip limit to set
*/
public void setSkipLimit(int skipLimit) {
public void setSkipLimit(long skipLimit) {
this.skipLimit = skipLimit;
}
@@ -124,7 +125,7 @@ public class LimitCheckingItemSkipPolicy implements SkipPolicy {
* {@link SkipLimitExceededException} will be thrown.
*/
@Override
public boolean shouldSkip(Throwable t, int skipCount) {
public boolean shouldSkip(Throwable t, long skipCount) {
if (skippableExceptionClassifier.classify(t)) {
if (skipCount < skipLimit) {
return true;

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2006-2013 the original author or authors.
* Copyright 2006-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,11 +21,12 @@ package org.springframework.batch.core.step.skip;
* indicating that an item should not be skipped.
*
* @author Lucas Ward
* @author Mahmoud Ben Hassine
*/
public class NeverSkipItemSkipPolicy implements SkipPolicy{
@Override
public boolean shouldSkip(Throwable t, int skipCount) {
public boolean shouldSkip(Throwable t, long skipCount) {
return false;
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2006-2007 the original author or authors.
* Copyright 2006-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -23,18 +23,19 @@ package org.springframework.batch.core.step.skip;
* @author Ben Hale
* @author Lucas Ward
* @author Dave Syer
* @author Mahmoud Ben Hassine
*/
@SuppressWarnings("serial")
public class SkipLimitExceededException extends SkipException {
private final int skipLimit;
private final long skipLimit;
public SkipLimitExceededException(int skipLimit, Throwable t) {
public SkipLimitExceededException(long skipLimit, Throwable t) {
super("Skip limit of '" + skipLimit + "' exceeded", t);
this.skipLimit = skipLimit;
}
public int getSkipLimit() {
public long getSkipLimit() {
return skipLimit;
}
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2006-2019 the original author or authors.
* Copyright 2006-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -38,6 +38,6 @@ public interface SkipPolicy {
* @throws SkipLimitExceededException if a limit is breached
* @throws IllegalArgumentException if the exception is null
*/
boolean shouldSkip(Throwable t, int skipCount) throws SkipLimitExceededException;
boolean shouldSkip(Throwable t, long skipCount) throws SkipLimitExceededException;
}

View File

@@ -272,7 +272,7 @@ public class FaultTolerantStepFactoryBeanTests {
factory.setSkipPolicy(new SkipPolicy() {
@Override
public boolean shouldSkip(Throwable t, int skipCount) throws SkipLimitExceededException {
public boolean shouldSkip(Throwable t, long skipCount) throws SkipLimitExceededException {
throw new RuntimeException("Planned exception in SkipPolicy");
}
});
@@ -300,7 +300,7 @@ public class FaultTolerantStepFactoryBeanTests {
factory.setSkipPolicy(new SkipPolicy() {
@Override
public boolean shouldSkip(Throwable t, int skipCount) throws SkipLimitExceededException {
public boolean shouldSkip(Throwable t, long skipCount) throws SkipLimitExceededException {
throw new RuntimeException("Planned exception in SkipPolicy");
}
});

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2006-2019 the original author or authors.
* Copyright 2006-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -96,10 +96,10 @@ public class FootballJobSkipIntegrationTests extends AbstractIntegrationTests {
if (stepExecution.getStepName().equals("playerload")) {
// The effect of the retries is to increase the number of
// rollbacks
int commitInterval = stepExecution.getReadCount() / (stepExecution.getCommitCount() - 1);
long commitInterval = stepExecution.getReadCount() / (stepExecution.getCommitCount() - 1);
// Account for the extra empty commit if the read count is
// commensurate with the commit interval
int effectiveCommitCount = stepExecution.getReadCount() % commitInterval == 0 ? stepExecution
long effectiveCommitCount = stepExecution.getReadCount() % commitInterval == 0 ? stepExecution
.getCommitCount() - 1 : stepExecution.getCommitCount();
long expectedRollbacks = Math.max(1, retryLimit) * effectiveCommitCount + stepExecution.getReadCount();
assertEquals(expectedRollbacks, stepExecution.getRollbackCount());

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2010-2019 the original author or authors.
* Copyright 2010-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -280,7 +280,7 @@ public class FaultTolerantStepIntegrationTests {
private class SkipIllegalArgumentExceptionSkipPolicy implements SkipPolicy {
@Override
public boolean shouldSkip(Throwable throwable, int skipCount)
public boolean shouldSkip(Throwable throwable, long skipCount)
throws SkipLimitExceededException {
return throwable instanceof IllegalArgumentException;
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2008-2019 the original author or authors.
* Copyright 2008-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -31,6 +31,7 @@ import org.springframework.util.Assert;
/**
* @author Dan Garrette
* @author Mahmoud Ben Hassine
* @since 2.0
*/
public class ErrorLogTasklet implements Tasklet, StepExecutionListener {
@@ -52,7 +53,7 @@ public class ErrorLogTasklet implements Tasklet, StepExecutionListener {
/**
* @return
*/
private int getSkipCount() {
private long getSkipCount() {
if (stepExecution == null || stepName == null) {
return 0;
}