From 0afcf3965d004a8e9ccd1f03093285c311e6280b Mon Sep 17 00:00:00 2001 From: Dave Syer Date: Wed, 24 Aug 2016 10:32:41 +0100 Subject: [PATCH] Add support for rolling averages and attributes in RetryStatistics Using these you can get a pretty decent (if basic) hystrix-compatible metrics stream. The "open" state of the circuit is exposed as a RetryContext attribute for that purpose. --- .../policy/CircuitBreakerRetryPolicy.java | 8 +- .../retry/stats/DefaultRetryStatistics.java | 9 +- .../stats/DefaultRetryStatisticsFactory.java | 42 +++++ .../stats/DefaultStatisticsRepository.java | 15 +- .../ExponentialAverageRetryStatistics.java | 143 ++++++++++++++++++ .../retry/stats/MutableRetryStatistics.java | 38 +++++ .../retry/stats/RetryStatisticsFactory.java | 27 ++++ .../retry/stats/StatisticsListener.java | 11 ++ .../retry/support/RetryTemplate.java | 4 + 9 files changed, 288 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/springframework/retry/stats/DefaultRetryStatisticsFactory.java create mode 100644 src/main/java/org/springframework/retry/stats/ExponentialAverageRetryStatistics.java create mode 100644 src/main/java/org/springframework/retry/stats/MutableRetryStatistics.java create mode 100644 src/main/java/org/springframework/retry/stats/RetryStatisticsFactory.java diff --git a/src/main/java/org/springframework/retry/policy/CircuitBreakerRetryPolicy.java b/src/main/java/org/springframework/retry/policy/CircuitBreakerRetryPolicy.java index 7b677c8..b4f8b0f 100644 --- a/src/main/java/org/springframework/retry/policy/CircuitBreakerRetryPolicy.java +++ b/src/main/java/org/springframework/retry/policy/CircuitBreakerRetryPolicy.java @@ -28,6 +28,8 @@ import org.springframework.retry.context.RetryContextSupport; */ public class CircuitBreakerRetryPolicy implements RetryPolicy { + public static final String CIRCUIT_OPEN = "circuite.open"; + private static Log logger = LogFactory.getLog(CircuitBreakerRetryPolicy.class); private final RetryPolicy delegate; @@ -122,9 +124,9 @@ public class CircuitBreakerRetryPolicy implements RetryPolicy { retryable = this.policy.canRetry(this.context); } else if (time < this.openWindow) { - if ((Boolean) getAttribute("open") == false) { + if ((Boolean) getAttribute(CIRCUIT_OPEN) == false) { logger.trace("Opening circuit"); - setAttribute("open", true); + setAttribute(CIRCUIT_OPEN, true); } this.start = System.currentTimeMillis(); return true; @@ -140,7 +142,7 @@ public class CircuitBreakerRetryPolicy implements RetryPolicy { if (logger.isTraceEnabled()) { logger.trace("Open: " + !retryable); } - setAttribute("open", !retryable); + setAttribute(CIRCUIT_OPEN, !retryable); return !retryable; } diff --git a/src/main/java/org/springframework/retry/stats/DefaultRetryStatistics.java b/src/main/java/org/springframework/retry/stats/DefaultRetryStatistics.java index 4e7240f..47b24ef 100644 --- a/src/main/java/org/springframework/retry/stats/DefaultRetryStatistics.java +++ b/src/main/java/org/springframework/retry/stats/DefaultRetryStatistics.java @@ -18,13 +18,15 @@ package org.springframework.retry.stats; import java.util.concurrent.atomic.AtomicInteger; +import org.springframework.core.AttributeAccessorSupport; import org.springframework.retry.RetryStatistics; /** * @author Dave Syer * */ -public class DefaultRetryStatistics implements RetryStatistics { +@SuppressWarnings("serial") +public class DefaultRetryStatistics extends AttributeAccessorSupport implements RetryStatistics, MutableRetryStatistics { private String name; private AtomicInteger startedCount = new AtomicInteger(); @@ -104,22 +106,27 @@ public class DefaultRetryStatistics implements RetryStatistics { this.abortCount.set(abortCount); } + @Override public void incrementStartedCount() { this.startedCount.incrementAndGet(); } + @Override public void incrementCompleteCount() { this.completeCount.incrementAndGet(); } + @Override public void incrementRecoveryCount() { this.recoveryCount.incrementAndGet(); } + @Override public void incrementErrorCount() { this.errorCount.incrementAndGet(); } + @Override public void incrementAbortCount() { this.abortCount.incrementAndGet(); } diff --git a/src/main/java/org/springframework/retry/stats/DefaultRetryStatisticsFactory.java b/src/main/java/org/springframework/retry/stats/DefaultRetryStatisticsFactory.java new file mode 100644 index 0000000..0b4e928 --- /dev/null +++ b/src/main/java/org/springframework/retry/stats/DefaultRetryStatisticsFactory.java @@ -0,0 +1,42 @@ +/* + * Copyright 2012-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.retry.stats; + +/** + * @author Dave Syer + * + */ +public class DefaultRetryStatisticsFactory implements RetryStatisticsFactory { + + private long window = 15000; + + /** + * Window in milliseconds for exponential decay factor in rolling averages. + * @param window the window to set + */ + public void setWindow(long window) { + this.window = window; + } + + @Override + public MutableRetryStatistics create(String name) { + ExponentialAverageRetryStatistics stats = new ExponentialAverageRetryStatistics(name); + stats.setWindow(window); + return stats; + } + +} diff --git a/src/main/java/org/springframework/retry/stats/DefaultStatisticsRepository.java b/src/main/java/org/springframework/retry/stats/DefaultStatisticsRepository.java index 33d3c20..1a5cc86 100644 --- a/src/main/java/org/springframework/retry/stats/DefaultStatisticsRepository.java +++ b/src/main/java/org/springframework/retry/stats/DefaultStatisticsRepository.java @@ -28,13 +28,18 @@ import org.springframework.retry.RetryStatistics; */ public class DefaultStatisticsRepository implements StatisticsRepository { - private ConcurrentMap map = new ConcurrentHashMap(); + private ConcurrentMap map = new ConcurrentHashMap(); + private RetryStatisticsFactory factory = new DefaultRetryStatisticsFactory(); + + public void setRetryStatisticsFactory(RetryStatisticsFactory factory) { + this.factory = factory; + } @Override public RetryStatistics findOne(String name) { return map.get(name); } - + @Override public Iterable findAll() { return new ArrayList(map.values()); @@ -65,10 +70,10 @@ public class DefaultStatisticsRepository implements StatisticsRepository { getStatistics(name).incrementAbortCount(); } - private DefaultRetryStatistics getStatistics(String name) { - DefaultRetryStatistics stats; + private MutableRetryStatistics getStatistics(String name) { + MutableRetryStatistics stats; if (!map.containsKey(name)) { - map.putIfAbsent(name, new DefaultRetryStatistics(name)); + map.putIfAbsent(name, factory.create(name)); } stats = map.get(name); return stats; diff --git a/src/main/java/org/springframework/retry/stats/ExponentialAverageRetryStatistics.java b/src/main/java/org/springframework/retry/stats/ExponentialAverageRetryStatistics.java new file mode 100644 index 0000000..4a1e1df --- /dev/null +++ b/src/main/java/org/springframework/retry/stats/ExponentialAverageRetryStatistics.java @@ -0,0 +1,143 @@ +/* + * Copyright 2012-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.retry.stats; + +/** + * @author Dave Syer + * + */ +@SuppressWarnings("serial") +public class ExponentialAverageRetryStatistics extends DefaultRetryStatistics { + + private long window = 15000; + + private ExponentialAverage started; + + private ExponentialAverage error; + + private ExponentialAverage complete; + + private ExponentialAverage recovery; + + private ExponentialAverage abort; + + public ExponentialAverageRetryStatistics(String name) { + super(name); + init(); + } + + private void init() { + started = new ExponentialAverage(window); + error = new ExponentialAverage(window); + complete = new ExponentialAverage(window); + abort = new ExponentialAverage(window); + recovery = new ExponentialAverage(window); + } + + /** + * Window in milliseconds for exponential decay factor in rolling average. + * + * @param window the window to set + */ + public void setWindow(long window) { + this.window = window; + init(); + } + + public int getRollingStartedCount() { + return (int) Math.round(started.getValue()); + } + + public int getRollingErrorCount() { + return (int) Math.round(error.getValue()); + } + + public int getRollingAbortCount() { + return (int) Math.round(abort.getValue()); + } + + public int getRollingRecoveryCount() { + return (int) Math.round(recovery.getValue()); + } + + public int getRollingCompleteCount() { + return (int) Math.round(complete.getValue()); + } + + public double getRollingErrorRate() { + if (Math.round(started.getValue())==0) { + return 0.; + } + return (abort.getValue() + recovery.getValue()) / started.getValue(); + } + + @Override + public void incrementStartedCount() { + super.incrementStartedCount(); + started.increment(); + } + + @Override + public void incrementCompleteCount() { + super.incrementCompleteCount(); + complete.increment(); + } + + @Override + public void incrementRecoveryCount() { + super.incrementRecoveryCount(); + recovery.increment(); + } + + @Override + public void incrementErrorCount() { + super.incrementErrorCount(); + error.increment(); + } + + @Override + public void incrementAbortCount() { + super.incrementAbortCount(); + abort.increment(); + } + + private class ExponentialAverage { + + private final double alpha; + + private volatile long lastTime = System.currentTimeMillis(); + + private volatile double value = 0; + + public ExponentialAverage(long window) { + alpha = 1. / window; + } + + public synchronized void increment() { + long time = System.currentTimeMillis(); + value = value * Math.exp(-alpha*(time - lastTime)) + 1; + lastTime = time; + } + + public double getValue() { + long time = System.currentTimeMillis(); + return value * Math.exp(-alpha*(time - lastTime)); + } + + } + +} diff --git a/src/main/java/org/springframework/retry/stats/MutableRetryStatistics.java b/src/main/java/org/springframework/retry/stats/MutableRetryStatistics.java new file mode 100644 index 0000000..16a85bc --- /dev/null +++ b/src/main/java/org/springframework/retry/stats/MutableRetryStatistics.java @@ -0,0 +1,38 @@ +/* + * Copyright 2012-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.retry.stats; + +import org.springframework.core.AttributeAccessor; +import org.springframework.retry.RetryStatistics; + +/** + * @author Dave Syer + * + */ +public interface MutableRetryStatistics extends RetryStatistics, AttributeAccessor { + + void incrementStartedCount(); + + void incrementCompleteCount(); + + void incrementRecoveryCount(); + + void incrementErrorCount(); + + void incrementAbortCount(); + +} \ No newline at end of file diff --git a/src/main/java/org/springframework/retry/stats/RetryStatisticsFactory.java b/src/main/java/org/springframework/retry/stats/RetryStatisticsFactory.java new file mode 100644 index 0000000..d1e33d9 --- /dev/null +++ b/src/main/java/org/springframework/retry/stats/RetryStatisticsFactory.java @@ -0,0 +1,27 @@ +/* + * Copyright 2012-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.retry.stats; + +/** + * @author Dave Syer + * + */ +public interface RetryStatisticsFactory { + + MutableRetryStatistics create(String name); + +} diff --git a/src/main/java/org/springframework/retry/stats/StatisticsListener.java b/src/main/java/org/springframework/retry/stats/StatisticsListener.java index 458a363..0181664 100644 --- a/src/main/java/org/springframework/retry/stats/StatisticsListener.java +++ b/src/main/java/org/springframework/retry/stats/StatisticsListener.java @@ -16,9 +16,12 @@ package org.springframework.retry.stats; +import org.springframework.core.AttributeAccessor; import org.springframework.retry.RetryCallback; import org.springframework.retry.RetryContext; +import org.springframework.retry.RetryStatistics; import org.springframework.retry.listener.RetryListenerSupport; +import org.springframework.retry.policy.CircuitBreakerRetryPolicy; /** * @author Dave Syer @@ -52,6 +55,14 @@ public class StatisticsListener extends RetryListenerSupport { else if (isClosed(context)) { repository.addComplete(name); } + RetryStatistics stats = repository.findOne(name); + if (stats instanceof AttributeAccessor) { + if (context.hasAttribute(CircuitBreakerRetryPolicy.CIRCUIT_OPEN)) { + ((AttributeAccessor) stats).setAttribute( + CircuitBreakerRetryPolicy.CIRCUIT_OPEN, + context.getAttribute(CircuitBreakerRetryPolicy.CIRCUIT_OPEN)); + } + } } } diff --git a/src/main/java/org/springframework/retry/support/RetryTemplate.java b/src/main/java/org/springframework/retry/support/RetryTemplate.java index 8861ae5..390ca7b 100644 --- a/src/main/java/org/springframework/retry/support/RetryTemplate.java +++ b/src/main/java/org/springframework/retry/support/RetryTemplate.java @@ -462,6 +462,10 @@ public class RetryTemplate implements RetryOperations { return doOpenInternal(retryPolicy, state); } + // Start with a clean slate for state that others may be inspecting + context.removeAttribute(RetryContext.CLOSED); + context.removeAttribute(RetryContext.EXHAUSTED); + context.removeAttribute(RetryContext.RECOVERED); return context; }