[#99] Basic Health Metrics

Changes following review
    - Reusing CounterService to automatically profit from Dropwizard if present
    - NoOp is the default impl for SpanReporterService
    - SpanReporterService has configurable metric names (it's enough to change the
    name to 'meter.a.b.c' to profit from Dropwizard's meters)

    Fixes gh-99
This commit is contained in:
Marcin Grzejszczak
2016-01-21 15:55:26 +01:00
parent 3a4554d8c1
commit 301c6b230a
16 changed files with 328 additions and 64 deletions

View File

@@ -18,10 +18,19 @@ package org.springframework.cloud.sleuth.autoconfig;
import java.util.Random;
import org.springframework.boot.actuate.metrics.CounterService;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingClass;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.sleuth.Sampler;
import org.springframework.cloud.sleuth.Tracer;
import org.springframework.cloud.sleuth.metric.CounterServiceBasedSpanReporterService;
import org.springframework.cloud.sleuth.metric.NoOpSpanReporterService;
import org.springframework.cloud.sleuth.metric.SleuthMetricProperties;
import org.springframework.cloud.sleuth.metric.SpanReporterService;
import org.springframework.cloud.sleuth.sampler.IsTracingSampler;
import org.springframework.cloud.sleuth.trace.DefaultTracer;
import org.springframework.context.ApplicationEventPublisher;
@@ -33,6 +42,7 @@ import org.springframework.context.annotation.Configuration;
*/
@Configuration
@ConditionalOnProperty(value="spring.sleuth.enabled", matchIfMissing=true)
@EnableConfigurationProperties
public class TraceAutoConfiguration {
@Bean
@@ -53,4 +63,37 @@ public class TraceAutoConfiguration {
ApplicationEventPublisher publisher) {
return new DefaultTracer(sampler, random, publisher);
}
@Bean
@ConditionalOnMissingBean
public SleuthMetricProperties sleuthMetricProperties() {
return new SleuthMetricProperties();
}
@Configuration
@ConditionalOnClass(CounterService.class)
@ConditionalOnMissingBean(SpanReporterService.class)
protected static class CounterServiceSpanReporterConfig {
@Bean
@ConditionalOnBean(CounterService.class)
public SpanReporterService spanReporterCounterService(CounterService counterService,
SleuthMetricProperties sleuthMetricProperties) {
return new CounterServiceBasedSpanReporterService(sleuthMetricProperties.getSpan().getAcceptedName(),
sleuthMetricProperties.getSpan().getDroppedName(), counterService);
}
@Bean
@ConditionalOnMissingBean(CounterService.class)
public SpanReporterService noOpSpanReporterCounterService() {
return new NoOpSpanReporterService();
}
}
@Bean
@ConditionalOnMissingClass("org.springframework.boot.actuate.metrics.CounterService")
@ConditionalOnMissingBean(SpanReporterService.class)
public SpanReporterService noOpSpanReporterCounterService() {
return new NoOpSpanReporterService();
}
}

View File

@@ -0,0 +1,35 @@
package org.springframework.cloud.sleuth.metric;
import org.springframework.boot.actuate.metrics.CounterService;
/**
* Service to operate on accepted and dropped spans statistics.
*
* @author Marcin Grzejszczak
*/
public class CounterServiceBasedSpanReporterService implements SpanReporterService {
private final String acceptedSpansMetricName;
private final String droppedSpansMetricName;
private final CounterService counterService;
public CounterServiceBasedSpanReporterService(String acceptedSpansMetricName,
String droppedSpansMetricName, CounterService counterService) {
this.acceptedSpansMetricName = acceptedSpansMetricName;
this.droppedSpansMetricName = droppedSpansMetricName;
this.counterService = counterService;
}
@Override
public void incrementAcceptedSpans(long quantity) {
for (int i = 0; i < quantity; i++) {
this.counterService.increment(this.acceptedSpansMetricName);
}
}
@Override
public void incrementDroppedSpans(long quantity) {
for (int i = 0; i < quantity; i++) {
this.counterService.increment(this.droppedSpansMetricName);
}
}
}

View File

@@ -0,0 +1,17 @@
package org.springframework.cloud.sleuth.metric;
/**
* Span reporting service that does nothing
*
* @author Marcin Grzejszczak
*/
public class NoOpSpanReporterService implements SpanReporterService {
public void incrementAcceptedSpans(long quantity) {
}
public void incrementDroppedSpans(long quantity) {
}
}

View File

@@ -0,0 +1,45 @@
package org.springframework.cloud.sleuth.metric;
import org.springframework.boot.context.properties.ConfigurationProperties;
/**
* Configuration properties for Sleuth related metrics
*
* @author Marcin Grzejszczak
*/
@ConfigurationProperties("spring.sleuth.metric")
public class SleuthMetricProperties {
private Span span = new Span();
public Span getSpan() {
return this.span;
}
public void setSpan(Span span) {
this.span = span;
}
public static class Span {
private String acceptedName = "counter.span.accepted";
private String droppedName = "counter.span.dropped";
public String getAcceptedName() {
return this.acceptedName;
}
public void setAcceptedName(String acceptedName) {
this.acceptedName = acceptedName;
}
public String getDroppedName() {
return this.droppedName;
}
public void setDroppedName(String droppedName) {
this.droppedName = droppedName;
}
}
}

View File

@@ -0,0 +1,21 @@
package org.springframework.cloud.sleuth.metric;
/**
* @author Marcin Grzejszczak
*/
public interface SpanReporterService {
/**
* Called when spans are submitted to SpanCollector for processing.
*
* @param quantity the number of spans accepted.
*/
void incrementAcceptedSpans(long quantity);
/**
* Called when spans become lost for any reason and won't be delivered to the target collector.
*
* @param quantity the number of spans dropped.
*/
void incrementDroppedSpans(long quantity);
}

View File

@@ -27,8 +27,7 @@ import static org.springframework.cloud.sleuth.assertions.SleuthAssertions.then;
public class TraceAsyncIntegrationTests {
@Autowired ClassPerformingAsyncLogic classPerformingAsyncLogic;
@Autowired
Tracer tracer;
@Autowired Tracer tracer;
@Test
public void should_set_span_on_an_async_annotated_method() {

View File

@@ -91,6 +91,12 @@
<artifactId>brave-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>${dropwizard-metrics.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@@ -23,6 +23,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.SpringApplicationConfiguration;
import org.springframework.boot.test.WebIntegrationTest;
import org.springframework.cloud.sleuth.metric.SpanReporterService;
import org.springframework.cloud.sleuth.zipkin.HttpZipkinSpanReporter;
import org.springframework.cloud.sleuth.zipkin.ZipkinProperties;
import org.springframework.cloud.sleuth.zipkin.ZipkinSpanReporter;
@@ -73,12 +74,14 @@ public class ZipkinTests extends AbstractIntegrationTest {
public static class WaitUntilZipkinIsUpConfig {
@Bean
@SneakyThrows
public ZipkinSpanReporter spanCollector(final ZipkinProperties zipkin) {
public ZipkinSpanReporter spanCollector(final ZipkinProperties zipkin,
final SpanReporterService spanReporterService) {
await().until(new Runnable() {
@Override
public void run() {
try {
WaitUntilZipkinIsUpConfig.this.getSpanCollector(zipkin);
WaitUntilZipkinIsUpConfig.this.getSpanCollector(zipkin,
spanReporterService);
}
catch (Exception e) {
log.error("Exception occurred while trying to connect to zipkin ["
@@ -87,11 +90,13 @@ public class ZipkinTests extends AbstractIntegrationTest {
}
}
});
return getSpanCollector(zipkin);
return getSpanCollector(zipkin, spanReporterService);
}
private ZipkinSpanReporter getSpanCollector(ZipkinProperties zipkin) {
return new HttpZipkinSpanReporter(zipkin.getBaseUrl(), zipkin.getFlushInterval());
private ZipkinSpanReporter getSpanCollector(ZipkinProperties zipkin,
SpanReporterService spanReporterService) {
return new HttpZipkinSpanReporter(zipkin.getBaseUrl(), zipkin.getFlushInterval(),
spanReporterService);
}
}
}

View File

@@ -52,10 +52,6 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>

View File

@@ -26,6 +26,7 @@ import org.springframework.boot.autoconfigure.web.ServerProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.client.discovery.DiscoveryClient;
import org.springframework.cloud.sleuth.Span;
import org.springframework.cloud.sleuth.metric.SpanReporterService;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.cloud.stream.config.ChannelBindingAutoConfiguration;
import org.springframework.context.annotation.Bean;
@@ -55,7 +56,7 @@ public class SleuthStreamAutoConfiguration {
@Bean
@GlobalChannelInterceptor(patterns = SleuthSource.OUTPUT, order = Ordered.HIGHEST_PRECEDENCE)
public ChannelInterceptor zipkinChannelInterceptor() {
public ChannelInterceptor zipkinChannelInterceptor(final SpanReporterService spanReporterService) {
// don't trace the tracer (suppress spans originating from our own source)
return new ChannelInterceptorAdapter() {
@Override
@@ -63,12 +64,28 @@ public class SleuthStreamAutoConfiguration {
return MessageBuilder.fromMessage(message)
.setHeader(Span.NOT_SAMPLED_NAME, "").build();
}
@Override
public void afterSendCompletion(Message<?> message, MessageChannel channel,
boolean sent, Exception ex) {
if (!(message.getPayload() instanceof Spans)) {
return;
}
Spans spans = (Spans) message.getPayload();
int spanNumber = spans.getSpans().size();
if (sent) {
spanReporterService.incrementAcceptedSpans(spanNumber);
} else {
spanReporterService.incrementDroppedSpans(spanNumber);
}
}
};
}
@Bean
public StreamSpanListener sleuthTracer(HostLocator endpointLocator) {
return new StreamSpanListener(endpointLocator);
public StreamSpanListener sleuthTracer(HostLocator endpointLocator,
SpanReporterService spanReporterService) {
return new StreamSpanListener(endpointLocator, spanReporterService);
}
@Configuration

View File

@@ -16,12 +16,6 @@
package org.springframework.cloud.sleuth.stream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import org.springframework.cloud.sleuth.Span;
import org.springframework.cloud.sleuth.event.ClientReceivedEvent;
import org.springframework.cloud.sleuth.event.ClientSentEvent;
@@ -29,11 +23,19 @@ import org.springframework.cloud.sleuth.event.ServerReceivedEvent;
import org.springframework.cloud.sleuth.event.ServerSentEvent;
import org.springframework.cloud.sleuth.event.SpanAcquiredEvent;
import org.springframework.cloud.sleuth.event.SpanReleasedEvent;
import org.springframework.cloud.sleuth.metric.SpanReporterService;
import org.springframework.context.event.EventListener;
import org.springframework.core.annotation.Order;
import org.springframework.integration.annotation.InboundChannelAdapter;
import org.springframework.integration.annotation.MessageEndpoint;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
/**
* A message source for spans. Also handles RPC flavoured annotations.
*
@@ -48,10 +50,12 @@ public class StreamSpanListener {
public static final String SERVER_SEND = "ss";
private Collection<Span> queue = new ConcurrentLinkedQueue<>();
private HostLocator endpointLocator;
private final HostLocator endpointLocator;
private final SpanReporterService spanReporterService;
public StreamSpanListener(HostLocator endpointLocator) {
public StreamSpanListener(HostLocator endpointLocator, SpanReporterService spanReporterService) {
this.endpointLocator = endpointLocator;
this.spanReporterService = spanReporterService;
}
public void setQueue(Collection<Span> queue) {
@@ -112,8 +116,11 @@ public class StreamSpanListener {
iterator.remove();
}
}
return result.isEmpty() ? null
: new Spans(this.endpointLocator.locate(result.get(0)), result);
if (result.isEmpty()) {
return null;
}
this.spanReporterService.incrementAcceptedSpans(result.size());
return new Spans(this.endpointLocator.locate(result.get(0)), result);
}
}

View File

@@ -16,16 +16,11 @@
package org.springframework.cloud.sleuth.stream;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.List;
import javax.annotation.PostConstruct;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.actuate.metrics.CounterService;
import org.springframework.boot.autoconfigure.PropertyPlaceholderAutoConfiguration;
import org.springframework.boot.test.SpringApplicationConfiguration;
import org.springframework.cloud.sleuth.Sampler;
@@ -49,6 +44,15 @@ import org.springframework.integration.annotation.ServiceActivator;
import org.springframework.messaging.Message;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.verify;
/**
* @author Dave Syer
*
@@ -57,17 +61,11 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@RunWith(SpringJUnit4ClassRunner.class)
public class StreamSpanListenerTests {
@Autowired
private Tracer tracer;
@Autowired
private ApplicationContext application;
@Autowired
private ZipkinTestConfiguration test;
@Autowired
StreamSpanListener listener;
@Autowired Tracer tracer;
@Autowired ApplicationContext application;
@Autowired ZipkinTestConfiguration test;
@Autowired StreamSpanListener listener;
@Autowired CounterService counterService;
@PostConstruct
public void init() {
@@ -106,12 +104,20 @@ public class StreamSpanListenerTests {
assertEquals(0, this.test.spans.size());
}
@Test
public void shouldIncreaseNumberOfAcceptedSpans() {
Span context = this.tracer.startTrace("foo");
this.tracer.close(context);
this.listener.poll();
verify(this.counterService, atLeastOnce()).increment(anyString());
}
@Configuration
@Import({ ZipkinTestConfiguration.class, SleuthStreamAutoConfiguration.class,
TestSupportBinderAutoConfiguration.class, ChannelBindingAutoConfiguration.class,
TraceAutoConfiguration.class, PropertyPlaceholderAutoConfiguration.class })
protected static class TestConfiguration {
}
protected static class TestConfiguration {}
@Configuration
@MessageEndpoint
@@ -131,6 +137,10 @@ public class StreamSpanListenerTests {
return new AlwaysSampler();
}
@Bean CounterService counterService() {
return Mockito.mock(CounterService.class);
}
@PostConstruct
public void init() {
this.listener.setQueue(this.spans);

View File

@@ -1,6 +1,7 @@
package org.springframework.cloud.sleuth.zipkin;
import lombok.extern.apachecommons.CommonsLog;
import org.springframework.cloud.sleuth.metric.SpanReporterService;
import zipkin.Codec;
import zipkin.Span;
@@ -32,14 +33,18 @@ public final class HttpZipkinSpanReporter
private final String url;
private final BlockingQueue<Span> pending = new LinkedBlockingQueue<>(1000);
private final Flusher flusher; // Nullable for testing
private final SpanReporterService spanReporterService;
/**
* @param baseUrl URL of the zipkin query server instance. Like: http://localhost:9411/
* @param flushInterval in seconds. 0 implies spans are {@link #flush() flushed} externally.
* @param spanReporterService service to count number of accepted / dropped spans
*/
public HttpZipkinSpanReporter(String baseUrl, int flushInterval) {
public HttpZipkinSpanReporter(String baseUrl, int flushInterval,
SpanReporterService spanReporterService) {
this.url = baseUrl + (baseUrl.endsWith("/") ? "" : "/") + "api/v1/spans";
this.flusher = flushInterval > 0 ? new Flusher(this, flushInterval) : null;
this.spanReporterService = spanReporterService;
}
/**
@@ -47,17 +52,19 @@ public final class HttpZipkinSpanReporter
*
* @param span Span, should not be <code>null</code>.
*/
@Override public void report(Span span) {
// TODO: metrics.incrementAcceptedSpans(1);
@Override
public void report(Span span) {
this.spanReporterService.incrementAcceptedSpans(1);
if (!this.pending.offer(span)) {
// TODO: metrics.incrementDroppedSpans(1);
this.spanReporterService.incrementDroppedSpans(1);
}
}
/**
* Calling this will flush any pending spans to the http transport on the current thread.
*/
@Override public void flush() {
@Override
public void flush() {
if (this.pending.isEmpty())
return;
List<Span> drained = new ArrayList<>(this.pending.size());
@@ -70,7 +77,7 @@ public final class HttpZipkinSpanReporter
// NOTE: https://github.com/openzipkin/zipkin-java/issues/66 will throw instead of return null.
if (json == null) {
log.debug("failed to encode spans, dropping them: " + drained);
// TODO: metrics.incrementDroppedSpans(spanCount);
this.spanReporterService.incrementDroppedSpans(drained.size());
return;
}
@@ -85,8 +92,7 @@ public final class HttpZipkinSpanReporter
"error POSTing spans to " + this.url + ": as json: " + new String(json,
UTF_8), e);
}
// TODO: metrics.incrementDroppedSpans(spanCount);
return;
this.spanReporterService.incrementDroppedSpans(drained.size());
}
}
@@ -102,7 +108,8 @@ public final class HttpZipkinSpanReporter
this.scheduler.scheduleWithFixedDelay(this, 0, flushInterval, SECONDS);
}
@Override public void run() {
@Override
public void run() {
try {
this.flushable.flush();
}
@@ -139,11 +146,12 @@ public final class HttpZipkinSpanReporter
* Requests a cease of delivery. There will be at most one in-flight request processing after this
* call returns.
*/
@Override public void close() {
@Override
public void close() {
if (this.flusher != null)
this.flusher.scheduler.shutdown();
// throw any outstanding spans on the floor
int dropped = this.pending.drainTo(new LinkedList<>());
// TODO: metrics.incrementDroppedSpans(dropped);
this.spanReporterService.incrementDroppedSpans(dropped);
}
}

View File

@@ -25,6 +25,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.web.ServerProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.client.discovery.DiscoveryClient;
import org.springframework.cloud.sleuth.metric.SpanReporterService;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -39,9 +40,10 @@ public class ZipkinAutoConfiguration {
@Bean
@ConditionalOnMissingBean(ZipkinSpanReporter.class)
public ZipkinSpanReporter reporter() {
public ZipkinSpanReporter reporter(SpanReporterService spanReporterService) {
ZipkinProperties zipkin = zipkinProperties();
return new HttpZipkinSpanReporter(zipkin.getBaseUrl(), zipkin.getFlushInterval());
return new HttpZipkinSpanReporter(zipkin.getBaseUrl(), zipkin.getFlushInterval(),
spanReporterService);
}
@Bean

View File

@@ -6,6 +6,8 @@ import okhttp3.mockwebserver.RecordedRequest;
import okhttp3.mockwebserver.SocketPolicy;
import org.junit.Rule;
import org.junit.Test;
import org.springframework.cloud.sleuth.metric.CounterServiceBasedSpanReporterService;
import org.springframework.cloud.sleuth.metric.SpanReporterService;
import zipkin.Codec;
import zipkin.Span;
@@ -16,10 +18,13 @@ import static org.assertj.core.api.Assertions.assertThat;
public class HttpZipkinSpanReporterTest {
@Rule public final MockWebServer server = new MockWebServer();
InMemorySpanCounter inMemorySpanCounter = new InMemorySpanCounter();
SpanReporterService spanReporterService = new CounterServiceBasedSpanReporterService("accepted", "dropped",
this.inMemorySpanCounter);
// set flush interval to 0 so that tests can drive flushing explicitly
HttpZipkinSpanReporter reporter = new HttpZipkinSpanReporter(
this.server.url("").toString(), 0);
this.server.url("").toString(), 0, this.spanReporterService);
@Test
public void reportDoesntDoIO() throws Exception {
@@ -32,8 +37,8 @@ public class HttpZipkinSpanReporterTest {
public void reportIncrementsAcceptedMetrics() throws Exception {
this.reporter.report(span(1L, "foo"));
// TODO: assertThat(metrics.acceptedSpans.get()).isEqualTo(1);
// TODO: assertThat(metrics.droppedSpans.get()).isZero();
assertThat(this.inMemorySpanCounter.getAcceptedSpans()).isEqualTo(1);
assertThat(this.inMemorySpanCounter.getDroppedSpans()).isZero();
}
@Test
@@ -41,8 +46,8 @@ public class HttpZipkinSpanReporterTest {
for (int i = 0; i < 1001; i++)
this.reporter.report(span(1L, "foo"));
// TODO: assertThat(metrics.acceptedSpans.get()).isEqualTo(1001);
// TODO: assertThat(metrics.droppedSpans.get()).isEqualTo(1);
assertThat(this.inMemorySpanCounter.getAcceptedSpans()).isEqualTo(1001);
assertThat(this.inMemorySpanCounter.getDroppedSpans()).isEqualTo(1);
}
@Test
@@ -73,7 +78,7 @@ public class HttpZipkinSpanReporterTest {
this.reporter.flush(); // manually flush the spans
// TODO: assertThat(metrics.droppedSpans.get()).isEqualTo(2);
assertThat(this.inMemorySpanCounter.getDroppedSpans()).isEqualTo(2);
}
@Test
@@ -86,7 +91,7 @@ public class HttpZipkinSpanReporterTest {
this.reporter.flush(); // manually flush the spans
// TODO: assertThat(metrics.droppedSpans.get()).isEqualTo(2);
assertThat(this.inMemorySpanCounter.getDroppedSpans()).isEqualTo(2);
}
static Span span(long traceId, String spanName) {

View File

@@ -0,0 +1,48 @@
package org.springframework.cloud.sleuth.zipkin;
import org.springframework.boot.actuate.metrics.CounterService;
import java.util.concurrent.atomic.AtomicLong;
/**
* Implementation of the metrics statistics held in-memory.
*
* @author Marcin Grzejszczak
*/
public class InMemorySpanCounter implements CounterService {
private final AtomicLong acceptedSpans = new AtomicLong(0);
private final AtomicLong droppedSpans = new AtomicLong(0);
public long getAcceptedSpans() {
return this.acceptedSpans.get();
}
public long getDroppedSpans() {
return this.droppedSpans.get();
}
@Override
public void increment(String metricName) {
if (metricName.contains("accepted")) {
this.acceptedSpans.incrementAndGet();
} else {
this.droppedSpans.incrementAndGet();
}
}
@Override
public void decrement(String metricName) {
if (metricName.contains("accepted")) {
this.acceptedSpans.decrementAndGet();
} else {
this.droppedSpans.decrementAndGet();
}
}
@Override
public void reset(String metricName) {
this.acceptedSpans.set(0);
this.droppedSpans.set(0);
}
}