Spring/spring-retry
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make ExponentialRandomBackOffPolicy more random

Browse Source 
The old version was still very prone to multiple threads marching
in lock step.  It is better to use random backoffs (with some
exponential growth).
This commit is contained in:
Dave Syer
2012-12-18 13:22:42 +00:00
parent 907bba3c2c
commit e9fa48236f
7 changed files with 157 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/main/java/org/springframework/retry/backoff/ExponentialRandomBackOffPolicy.java
View File

@@ -40,6 +40,7 @@ import java.util.Random;
* {@link ExponentialRandomBackOffPolicy} may yield [50, 100, 100, 100, 600]
* or [50, 100, 150, 400, 800]
* @author Jon Travis
* @author Dave Syer
*/
public class ExponentialRandomBackOffPolicy extends ExponentialBackOffPolicy {
/**
@@ -56,19 +57,17 @@ public class ExponentialRandomBackOffPolicy extends ExponentialBackOffPolicy {
static class ExponentialRandomBackOffContext extends ExponentialBackOffPolicy.ExponentialBackOffContext {
private final Random r = new Random();
private final long initialInterval;
private long intervalIdx;
public ExponentialRandomBackOffContext(long expSeed, double multiplier, long maxInterval) {
super(expSeed, multiplier, maxInterval);
this.initialInterval = expSeed;
this.intervalIdx = 0;
}
@Override
public synchronized long getSleepAndIncrement() {
long next = super.getSleepAndIncrement();
next = (long)(next*(1 + r.nextFloat()*(getMultiplier()-1)));
return next;
}
@Override
protected synchronized long getNextInterval() {
intervalIdx++;
return initialInterval + initialInterval * Math.max(1, r.nextInt((int)Math.pow(getMultiplier(), intervalIdx)));
}
}
}

2
src/main/java/org/springframework/retry/backoff/SleepingBackOffPolicy.java
View File

@@ -20,7 +20,7 @@ package org.springframework.retry.backoff;
* A interface which can be mixed in by {@link BackOffPolicy}s indicating that they sleep
* when backing off.
*/
public interface SleepingBackOffPolicy<T extends SleepingBackOffPolicy> extends BackOffPolicy {
public interface SleepingBackOffPolicy<T extends SleepingBackOffPolicy<T>> extends BackOffPolicy {
/**
* Clone the policy and return a new policy which uses the passed sleeper.
*

158
src/main/java/org/springframework/retry/support/RetrySimulation.java
View File

@@ -16,99 +16,99 @@
package org.springframework.retry.support;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* The results of a simulation.
*/
public class RetrySimulation {
private final List<SleepSequence> sleepSequences = new ArrayList<SleepSequence>();
private final Map<Long, Long> sleepHistogram = new HashMap<Long, Long>();
/**
* Add a sequence of sleeps to the simulation.
*/
public void addSequence(List<Long> sleeps) {
for (Long sleep : sleeps) {
Long existingHisto = sleepHistogram.get(sleep);
if (existingHisto == null) {
sleepHistogram.put(sleep, 1l);
} else {
sleepHistogram.put(sleep, existingHisto + 1);
}
}
private final List<SleepSequence> sleepSequences = new ArrayList<SleepSequence>();
sleepSequences.add(new SleepSequence(sleeps));
}
private final List<Long> sleepHistogram = new ArrayList<Long>();
/**
* @return Returns a list of all the unique sleep values which were executed within
* all simulations.
*/
public List<Long> getUniqueSleeps() {
List<Long> res = new ArrayList<Long>(sleepHistogram.keySet());
Collections.sort(res);
return res;
}
public RetrySimulation() {
}
/**
* @return the count of each sleep which was seen throughout all sleeps.
* histogram[i] = sum(getUniqueSleeps()[i])
*/
public List<Long> getUniqueSleepsHistogram() {
List<Long> res = new ArrayList<Long>(sleepHistogram.size());
for (Long sleep : getUniqueSleeps()) {
res.add(sleepHistogram.get(sleep));
}
return res;
}
/**
* Add a sequence of sleeps to the simulation.
*/
public void addSequence(List<Long> sleeps) {
sleepHistogram.addAll(sleeps);
sleepSequences.add(new SleepSequence(sleeps));
}
/**
* @return the longest total time slept by a retry sequence.
*/
public SleepSequence getLongestTotalSleepSequence() {
SleepSequence longest = null;
for (SleepSequence sequence : sleepSequences) {
if (longest == null || sequence.getTotalSleep() > longest.getTotalSleep()) {
longest = sequence;
}
}
return longest;
}
/**
* @return Returns a list of all the unique sleep values which were executed within all simulations.
*/
public List<Double> getPercentiles() {
List<Double> res = new ArrayList<Double>();
for (double percentile : new double[] { 10, 20, 30, 40, 50, 60, 70, 80, 90 }) {
res.add(getPercentile(percentile / 100));
}
return res;
}
public static class SleepSequence {
private final List<Long> sleeps;
private final long longestSleep;
private final long totalSleep;
public double getPercentile(double p) {
Collections.sort(sleepHistogram);
int size = sleepHistogram.size();
double pos = p * (size - 1);
int i0 = (int) pos;
int i1 = i0 + 1;
double weight = pos - i0;
return sleepHistogram.get(i0) * (1 - weight) + sleepHistogram.get(i1) * weight;
public SleepSequence(List<Long> sleeps) {
this.sleeps = sleeps;
this.longestSleep = Collections.max(sleeps);
long totalSleep = 0;
for (Long sleep : sleeps) {
totalSleep += sleep;
}
this.totalSleep = totalSleep;
}
}
public List<Long> getSleeps() {
return sleeps;
}
/**
* @return the longest total time slept by a retry sequence.
*/
public SleepSequence getLongestTotalSleepSequence() {
SleepSequence longest = null;
for (SleepSequence sequence : sleepSequences) {
if (longest == null || sequence.getTotalSleep() > longest.getTotalSleep()) {
longest = sequence;
}
}
return longest;
}
/**
* Returns the longest individual sleep within this sequence.
*/
public long getLongestSleep() {
return longestSleep;
}
public static class SleepSequence {
private final List<Long> sleeps;
public long getTotalSleep() {
return totalSleep;
}
private final long longestSleep;
public String toString() {
return "totalSleep=" + totalSleep + ": " + sleeps.toString();
}
}
private final long totalSleep;
public SleepSequence(List<Long> sleeps) {
this.sleeps = sleeps;
this.longestSleep = Collections.max(sleeps);
long totalSleep = 0;
for (Long sleep : sleeps) {
totalSleep += sleep;
}
this.totalSleep = totalSleep;
}
public List<Long> getSleeps() {
return sleeps;
}
/**
* Returns the longest individual sleep within this sequence.
*/
public long getLongestSleep() {
return longestSleep;
}
public long getTotalSleep() {
return totalSleep;
}
public String toString() {
return "totalSleep=" + totalSleep + ": " + sleeps.toString();
}
}
}

17
src/main/java/org/springframework/retry/support/RetrySimulator.java
View File

@@ -16,15 +16,15 @@
package org.springframework.retry.support;
import java.util.ArrayList;
import java.util.List;
import org.springframework.retry.RetryCallback;
import org.springframework.retry.RetryContext;
import org.springframework.retry.RetryPolicy;
import org.springframework.retry.backoff.Sleeper;
import org.springframework.retry.backoff.SleepingBackOffPolicy;
import java.util.ArrayList;
import java.util.List;
/**
* A {@link RetrySimulator} is a tool for exercising retry + backoff operations.
*
@@ -49,10 +49,11 @@ import java.util.List;
* @author Jon Travis
*/
public class RetrySimulator {
private final SleepingBackOffPolicy backOffPolicy;
private final SleepingBackOffPolicy<?> backOffPolicy;
private final RetryPolicy retryPolicy;
public RetrySimulator(SleepingBackOffPolicy backOffPolicy, RetryPolicy retryPolicy) {
public RetrySimulator(SleepingBackOffPolicy<?> backOffPolicy, RetryPolicy retryPolicy) {
this.backOffPolicy = backOffPolicy;
this.retryPolicy = retryPolicy;
}
@@ -78,14 +79,14 @@ public class RetrySimulator {
*/
public List<Long> executeSingleSimulation() {
StealingSleeper stealingSleeper = new StealingSleeper();
SleepingBackOffPolicy stealingBackoff = backOffPolicy.withSleeper(stealingSleeper);
SleepingBackOffPolicy<?> stealingBackoff = backOffPolicy.withSleeper(stealingSleeper);
RetryTemplate template = new RetryTemplate();
template.setBackOffPolicy(stealingBackoff);
template.setRetryPolicy(retryPolicy);
try {
template.execute(new FailingRetryCallback<Object>());
template.execute(new FailingRetryCallback());
} catch(FailingRetryException e) {
} catch(Exception e) {
@@ -95,7 +96,7 @@ public class RetrySimulator {
return stealingSleeper.getSleeps();
}
static class FailingRetryCallback<Object> implements RetryCallback<Object> {
static class FailingRetryCallback implements RetryCallback<Object> {
public Object doWithRetry(RetryContext context) throws Exception {
throw new FailingRetryException();
}

23
src/test/java/org/springframework/retry/backoff/ExponentialRandomBackOffPolicyTests.java
View File

@@ -16,15 +16,14 @@
package org.springframework.retry.backoff;
import java.util.List;
import junit.framework.TestCase;
import org.springframework.retry.RetryContext;
import org.springframework.retry.RetryPolicy;
import org.springframework.retry.policy.SimpleRetryPolicy;
import org.springframework.retry.support.RetrySimulation;
import org.springframework.retry.support.RetrySimulator;
import java.util.*;
public class ExponentialRandomBackOffPolicyTests extends TestCase {
static final int NUM_TRIALS = 10000;
static final int MAX_RETRIES = 6;
@@ -53,9 +52,7 @@ public class ExponentialRandomBackOffPolicyTests extends TestCase {
assertEquals(MAX_RETRIES - 1, sleeps.size());
long initialInterval = backOffPolicy.getInitialInterval();
for (int i=0; i<sleeps.size(); i++) {
assertTrue(sleeps.get(i) % backOffPolicy.getInitialInterval() == 0);
long expectedMaxValue = (long) (initialInterval + initialInterval * Math.max(1, Math.pow(backOffPolicy.getMultiplier(), i)));
long expectedMaxValue = 2*(long) (initialInterval + initialInterval * Math.max(1, Math.pow(backOffPolicy.getMultiplier(), i)));
assertTrue("Found a sleep [" + sleeps.get(i) + "] which exceeds our max expected value of " + expectedMaxValue + " at interval " + i,
sleeps.get(i) < expectedMaxValue);
}
@@ -69,18 +66,10 @@ public class ExponentialRandomBackOffPolicyTests extends TestCase {
System.out.println("Ran " + NUM_TRIALS + " backoff trials. Each trial retried " + MAX_RETRIES + " times");
System.out.println("Policy: " + backOffPolicy);
System.out.println("All generated backoffs:");
System.out.println(" " + simulation.getUniqueSleeps());
System.out.println(" " + simulation.getPercentiles());
System.out.println("Backoff frequencies:");
System.out.print(" " + simulation.getUniqueSleepsHistogram());
System.out.print(" " + simulation.getPercentiles());
long expectedSleep = backOffPolicy.getInitialInterval();
List<Long> allSleeps = simulation.getUniqueSleeps();
for (int j=0; j<allSleeps.size(); j++) {
long sleep = allSleeps.get(j);
assertEquals("Missing a multiple of " + backOffPolicy.getInitialInterval(), sleep, expectedSleep);
expectedSleep += backOffPolicy.getInitialInterval();
}
}
}

111
src/test/java/org/springframework/retry/support/RetrySimulationTests.java
View File

@@ -16,81 +16,74 @@
package org.springframework.retry.support;
import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
import org.springframework.retry.backoff.ExponentialBackOffPolicy;
import org.springframework.retry.backoff.ExponentialRandomBackOffPolicy;
import org.springframework.retry.backoff.FixedBackOffPolicy;
import org.springframework.retry.policy.SimpleRetryPolicy;
import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class RetrySimulationTests {
@Test
public void testSimulatorExercisesFixedBackoff() {
SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy();
retryPolicy.setMaxAttempts(5);
FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy();
backOffPolicy.setBackOffPeriod(400);
@Test
public void testSimulatorExercisesFixedBackoff() {
SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy();
retryPolicy.setMaxAttempts(5);
RetrySimulator simulator = new RetrySimulator(backOffPolicy, retryPolicy);
RetrySimulation simulation = simulator.executeSimulation(1000);
System.out.println(backOffPolicy);
System.out.println("Longest sequence " + simulation.getLongestTotalSleepSequence());
System.out.println("All Sleeps: " + simulation.getUniqueSleeps());
System.out.println("Sleep Occurences: " + simulation.getUniqueSleepsHistogram());
FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy();
backOffPolicy.setBackOffPeriod(400);
assertEquals(asList(400l, 400l, 400l, 400l), simulation.getLongestTotalSleepSequence().getSleeps());
assertEquals(asList(400l), simulation.getUniqueSleeps());
assertEquals(asList(4000l), simulation.getUniqueSleepsHistogram());
}
RetrySimulator simulator = new RetrySimulator(backOffPolicy, retryPolicy);
RetrySimulation simulation = simulator.executeSimulation(1000);
System.out.println(backOffPolicy);
System.out.println("Longest sequence " + simulation.getLongestTotalSleepSequence());
System.out.println("Percentiles: " + simulation.getPercentiles());
@Test
public void testSimulatorExercisesExponentialBackoff() {
SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy();
retryPolicy.setMaxAttempts(5);
assertEquals(asList(400l, 400l, 400l, 400l), simulation.getLongestTotalSleepSequence().getSleeps());
assertEquals(asList(400d, 400d, 400d, 400d, 400d, 400d, 400d, 400d, 400d), simulation.getPercentiles());
assertEquals(400d, simulation.getPercentile(0.5),0.1);
}
ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
backOffPolicy.setMultiplier(2);
backOffPolicy.setMaxInterval(30000);
backOffPolicy.setInitialInterval(100);
@Test
public void testSimulatorExercisesExponentialBackoff() {
SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy();
retryPolicy.setMaxAttempts(5);
RetrySimulator simulator = new RetrySimulator(backOffPolicy, retryPolicy);
RetrySimulation simulation = simulator.executeSimulation(1000);
System.out.println(backOffPolicy);
System.out.println("Longest sequence " + simulation.getLongestTotalSleepSequence());
System.out.println("All Sleeps: " + simulation.getUniqueSleeps());
System.out.println("Sleep Occurences: " + simulation.getUniqueSleepsHistogram());
ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
backOffPolicy.setMultiplier(2);
backOffPolicy.setMaxInterval(30000);
backOffPolicy.setInitialInterval(100);
assertEquals(asList(100l, 200l, 400l, 800l), simulation.getLongestTotalSleepSequence().getSleeps());
assertEquals(asList(100l, 200l, 400l, 800l), simulation.getUniqueSleeps());
assertEquals(asList(1000l, 1000l, 1000l, 1000l), simulation.getUniqueSleepsHistogram());
}
RetrySimulator simulator = new RetrySimulator(backOffPolicy, retryPolicy);
RetrySimulation simulation = simulator.executeSimulation(1000);
System.out.println(backOffPolicy);
System.out.println("Longest sequence " + simulation.getLongestTotalSleepSequence());
System.out.println("Percentiles: " + simulation.getPercentiles());
@Test
public void testSimulatorExercisesRandomExponentialBackoff() {
SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy();
retryPolicy.setMaxAttempts(5);
assertEquals(asList(100l, 200l, 400l, 800l), simulation.getLongestTotalSleepSequence().getSleeps());
assertEquals(asList(100d, 100d, 200d, 200d, 300d, 400d, 400d, 800d, 800d), simulation.getPercentiles());
assertEquals(300d, simulation.getPercentile(0.5f), 0.1);
}
ExponentialBackOffPolicy backOffPolicy = new ExponentialRandomBackOffPolicy();
backOffPolicy.setMultiplier(2);
backOffPolicy.setMaxInterval(30000);
backOffPolicy.setInitialInterval(100);
@Test
public void testSimulatorExercisesRandomExponentialBackoff() {
SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy();
retryPolicy.setMaxAttempts(5);
RetrySimulator simulator = new RetrySimulator(backOffPolicy, retryPolicy);
RetrySimulation simulation = simulator.executeSimulation(10000);
System.out.println(backOffPolicy);
System.out.println("Longest sequence " + simulation.getLongestTotalSleepSequence());
System.out.println("All Sleeps: " + simulation.getUniqueSleeps());
System.out.println("Sleep Occurences: " + simulation.getUniqueSleepsHistogram());
ExponentialBackOffPolicy backOffPolicy = new ExponentialRandomBackOffPolicy();
backOffPolicy.setMultiplier(2);
backOffPolicy.setMaxInterval(30000);
backOffPolicy.setInitialInterval(100);
assertEquals(asList(100l, 200l, 400l, 800l), simulation.getLongestTotalSleepSequence().getSleeps());
assertEquals(asList(100l, 200l, 300l, 400l, 500l, 600l, 700l, 800l), simulation.getUniqueSleeps());
for (long histo : simulation.getUniqueSleepsHistogram()) {
assertTrue("Should have experienced a sleep more than " + histo + " times",
histo > 1000);
}
}
RetrySimulator simulator = new RetrySimulator(backOffPolicy, retryPolicy);
RetrySimulation simulation = simulator.executeSimulation(10000);
System.out.println(backOffPolicy);
System.out.println("Longest sequence " + simulation.getLongestTotalSleepSequence());
System.out.println("Percentiles: " + simulation.getPercentiles());
assertTrue(simulation.getPercentiles().size() > 4);
}
}

2
template.mf
View File

@@ -5,6 +5,7 @@ Bundle-ManifestVersion: 2
Import-Template:
org.springframework.beans.*;version="[3.0.0, 4.0.0)",
org.springframework.context.*;version="[3.0.0, 4.0.0)",
org.springframework.aop.*;version="[3.0.0, 4.0.0)",
org.springframework.core.*;version="[3.0.0, 4.0.0)",
org.springframework.jdbc.*;version="[3.0.0, 4.0.0)",
org.springframework.stereotype.*;version="[3.0.0, 4.0.0)",
@@ -14,4 +15,5 @@ Import-Template:
org.springframework.web.*;version="[3.0.0, 4.0.0)",
org.springframework.validation.*;version="[3.0.0, 4.0.0)",
org.springframework.util;version="[3.0.0, 4.0.0)",
org.aopalliance.*;version="[1.0.0, 2.0.0)",
org.apache.commons.logging;version="[1.1.1, 2.0.0)"