Add an option to use the X-B3-Flags header to override any sampling decision

without this change it's pretty much impossible to enforce sampling for certain traces
    with this change setting the X-B3-Flags to 1 for HTTP messages / spanFlags to 1 for messaging will override any sampling decisions

    fixes #496
This commit is contained in:
Marcin Grzejszczak
2017-01-18 13:12:13 +01:00
parent 9785bcae15
commit c347f724cd
12 changed files with 344 additions and 21 deletions

View File

@@ -49,6 +49,10 @@ A sampler can be installed just by creating a bean definition, e.g:
include::../../../../spring-cloud-sleuth-core/src/test/java/org/springframework/cloud/sleuth/documentation/SpringCloudSleuthDocTests.java[tags=always_sampler,indent=0]
----
TIP: You can set the HTTP header `X-B3-Flags` to `1` or when doing messaging you can
set `spanFlags` header to `1`. Then the current span will be forced to be exportable
regardless of the sampling decision.
== Instrumentation
Spring Cloud Sleuth instruments all your Spring application

View File

@@ -28,13 +28,13 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
/**
* Class for gathering and reporting statistics about a block of execution.
* <p>
@@ -82,6 +82,7 @@ public class Span implements SpanContext {
public static final String SPAN_NAME_NAME = "X-Span-Name";
public static final String SPAN_ID_NAME = "X-B3-SpanId";
public static final String SPAN_EXPORT_NAME = "X-Span-Export";
public static final String SPAN_FLAGS = "X-B3-Flags";
public static final String SPAN_BAGGAGE_HEADER_PREFIX = "baggage";
public static final Set<String> SPAN_HEADERS = new HashSet<>(
Arrays.asList(SAMPLED_NAME, PROCESS_ID_NAME, PARENT_ID_NAME, TRACE_ID_NAME,

View File

@@ -1,6 +1,7 @@
package org.springframework.cloud.sleuth.instrument.messaging;
import java.util.Map;
import java.util.Random;
import org.springframework.cloud.sleuth.Span;
import org.springframework.cloud.sleuth.SpanTextMap;
@@ -17,7 +18,12 @@ public class HeaderBasedMessagingExtractor implements MessagingSpanTextMapExtrac
@Override
public Span joinTrace(SpanTextMap textMap) {
Map<String, String> carrier = TextMapUtil.asMap(textMap);
if (!hasHeader(carrier, TraceMessageHeaders.SPAN_ID_NAME)
if (Span.SPAN_SAMPLED.equals(carrier.get(TraceMessageHeaders.SPAN_FLAGS_NAME))) {
String traceId = generateTraceIdIfMissing(carrier);
if (!carrier.containsKey(TraceMessageHeaders.SPAN_ID_NAME)) {
carrier.put(TraceMessageHeaders.SPAN_ID_NAME, traceId);
}
} else if (!hasHeader(carrier, TraceMessageHeaders.SPAN_ID_NAME)
|| !hasHeader(carrier, TraceMessageHeaders.TRACE_ID_NAME)) {
return null;
// TODO: Consider throwing IllegalArgumentException;
@@ -25,14 +31,26 @@ public class HeaderBasedMessagingExtractor implements MessagingSpanTextMapExtrac
return extractSpanFromHeaders(carrier, Span.builder());
}
private String generateTraceIdIfMissing(Map<String, String> carrier) {
if (!hasHeader(carrier, TraceMessageHeaders.TRACE_ID_NAME)) {
carrier.put(TraceMessageHeaders.TRACE_ID_NAME, Span.idToHex(new Random().nextLong()));
}
return carrier.get(TraceMessageHeaders.TRACE_ID_NAME);
}
private Span extractSpanFromHeaders(Map<String, String> carrier, Span.SpanBuilder spanBuilder) {
String traceId = carrier.get(TraceMessageHeaders.TRACE_ID_NAME);
spanBuilder = spanBuilder
.traceIdHigh(traceId.length() == 32 ? Span.hexToId(traceId, 0) : 0)
.traceId(Span.hexToId(traceId))
.spanId(Span.hexToId(carrier.get(TraceMessageHeaders.SPAN_ID_NAME)));
spanBuilder.exportable(
String flags = carrier.get(TraceMessageHeaders.SPAN_FLAGS_NAME);
if (Span.SPAN_SAMPLED.equals(flags)) {
spanBuilder.exportable(true);
} else {
spanBuilder.exportable(
Span.SPAN_SAMPLED.equals(carrier.get(TraceMessageHeaders.SAMPLED_NAME)));
}
String processId = carrier.get(TraceMessageHeaders.PROCESS_ID_NAME);
String spanName = carrier.get(TraceMessageHeaders.SPAN_NAME_NAME);
if (spanName != null) {

View File

@@ -32,6 +32,7 @@ public class TraceMessageHeaders {
public static final String PARENT_ID_NAME = "spanParentSpanId";
public static final String TRACE_ID_NAME = "spanTraceId";
public static final String SPAN_NAME_NAME = "spanName";
public static final String SPAN_FLAGS_NAME = "spanFlags";
static final String MESSAGE_SENT_FROM_CLIENT = "messageSent";
static final String HEADER_DELIMITER = "_";

View File

@@ -15,18 +15,18 @@
*/
package org.springframework.cloud.sleuth.instrument.web;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
import java.util.regex.Pattern;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -34,6 +34,7 @@ import org.springframework.cloud.sleuth.Span;
import org.springframework.cloud.sleuth.SpanReporter;
import org.springframework.cloud.sleuth.TraceKeys;
import org.springframework.cloud.sleuth.Tracer;
import org.springframework.cloud.sleuth.sampler.AlwaysSampler;
import org.springframework.cloud.sleuth.sampler.NeverSampler;
import org.springframework.cloud.sleuth.util.ExceptionUtils;
import org.springframework.core.Ordered;
@@ -311,7 +312,12 @@ public class TraceFilter extends GenericFilterBean {
spanFromRequest = this.tracer.createSpan(name, NeverSampler.INSTANCE);
}
else {
spanFromRequest = this.tracer.createSpan(name);
String header = request.getHeader(Span.SPAN_FLAGS);
if (Span.SPAN_SAMPLED.equals(header)) {
spanFromRequest = this.tracer.createSpan(name, new AlwaysSampler());
} else {
spanFromRequest = this.tracer.createSpan(name);
}
}
spanFromRequest.logEvent(Span.SERVER_RECV);
request.setAttribute(TRACE_REQUEST_ATTR, spanFromRequest);

View File

@@ -2,6 +2,7 @@ package org.springframework.cloud.sleuth.instrument.web;
import java.lang.invoke.MethodHandles;
import java.util.Map;
import java.util.Random;
import java.util.regex.Pattern;
import org.apache.commons.logging.LogFactory;
@@ -33,7 +34,12 @@ public class ZipkinHttpSpanExtractor implements HttpSpanExtractor {
@Override
public Span joinTrace(SpanTextMap textMap) {
Map<String, String> carrier = TextMapUtil.asMap(textMap);
if (carrier.get(Span.TRACE_ID_NAME) == null) {
boolean debug = Span.SPAN_SAMPLED.equals(carrier.get(Span.SPAN_FLAGS));
if (debug) {
// we're only generating Trace ID since if there's no Span ID will assume
// that it's equal to Trace ID
generateIdIfMissing(carrier, Span.TRACE_ID_NAME);
} else if (carrier.get(Span.TRACE_ID_NAME) == null) {
// can't build a Span without trace id
return null;
}
@@ -49,6 +55,12 @@ public class ZipkinHttpSpanExtractor implements HttpSpanExtractor {
}
}
private void generateIdIfMissing(Map<String, String> carrier, String key) {
if (!carrier.containsKey(key)) {
carrier.put(key, Span.idToHex(new Random().nextLong()));
}
}
private long spanId(Map<String, String> carrier) {
String spanId = carrier.get(Span.SPAN_ID_NAME);
if (spanId == null) {
@@ -82,7 +94,10 @@ public class ZipkinHttpSpanExtractor implements HttpSpanExtractor {
span.parent(Span.hexToId(carrier.get(Span.PARENT_ID_NAME)));
}
span.remote(true);
if (skip) {
boolean debug = Span.SPAN_SAMPLED.equals(carrier.get(Span.SPAN_FLAGS));
if (debug) {
span.exportable(true);
} else if (skip) {
span.exportable(false);
}
for (Map.Entry<String, String> entry : carrier.entrySet()) {

View File

@@ -16,16 +16,11 @@
package org.springframework.cloud.sleuth.assertions;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.assertj.core.api.AbstractAssert;
@@ -39,8 +34,6 @@ public class ListOfSpansAssert extends AbstractAssert<ListOfSpansAssert, ListOfS
private static final Log log = LogFactory.getLog(ListOfSpansAssert.class);
private final ObjectMapper objectMapper = new ObjectMapper();
public ListOfSpansAssert(ListOfSpans actual) {
super(actual, ListOfSpansAssert.class);
}
@@ -111,6 +104,26 @@ public class ListOfSpansAssert extends AbstractAssert<ListOfSpansAssert, ListOfS
return this;
}
public ListOfSpansAssert allSpansAreExportable() {
isNotNull();
printSpans();
if (!everySpanIsExportable()) {
failWithMessage("Expected spans \n <%s> \nto be exportable but there's at least "
+ "one which is not", spansToString());
}
return this;
}
public ListOfSpansAssert allSpansHaveTraceId(long traceId) {
isNotNull();
printSpans();
if (!everySpanHasTraceId(traceId)) {
failWithMessage("Expected spans \n <%s> \nto have trace id <%s> but there's at least "
+ "one which doesn't have it", spansToString(), traceId);
}
return this;
}
private boolean spanWithKeyTagExists(String tagKey) {
for (Span span : this.actual.spans) {
if (span.tags().containsKey(tagKey)) {
@@ -138,6 +151,24 @@ public class ListOfSpansAssert extends AbstractAssert<ListOfSpansAssert, ListOfS
return exists;
}
private boolean everySpanIsExportable() {
for (Span span : this.actual.spans) {
if (!span.isExportable()) {
return false;
}
}
return true;
}
private boolean everySpanHasTraceId(long traceId) {
for (Span span : this.actual.spans) {
if (span.getTraceId() != traceId) {
return false;
}
}
return true;
}
private boolean hasBaggage(String baggageKey, String baggageValue) {
for (Span span : this.actual.spans) {
for (Map.Entry<String, String> baggage : span.baggageItems()) {
@@ -186,6 +217,12 @@ public class ListOfSpansAssert extends AbstractAssert<ListOfSpansAssert, ListOfS
.collect(toList());
}
private List<Span> findSpansWithSpanId(long spanId) {
return this.actual.spans.stream()
.filter(span -> spanId == span.getSpanId())
.collect(toList());
}
public ListOfSpansAssert hasASpanWithName(String name) {
isNotNull();
printSpans();
@@ -196,6 +233,25 @@ public class ListOfSpansAssert extends AbstractAssert<ListOfSpansAssert, ListOfS
return this;
}
public ListOfSpansAssert hasASpanWithSpanId(Long spanId) {
isNotNull();
printSpans();
List<Span> matchingSpans = findSpansWithSpanId(spanId);
if (matchingSpans.isEmpty()) {
failWithMessage("Expected spans <%s> to contain a span with id <%s>", spansToString(), spanId);
}
return this;
}
public ListOfSpansAssert hasSize(int size) {
isNotNull();
printSpans();
if (size != this.actual.spans.size()) {
failWithMessage("Expected spans <%s> to be of size <%s> but was <%s>", spansToString(), size, actual.spans.size());
}
return this;
}
private void printSpans() {
log.info("Stored spans " + spansToString());
}

View File

@@ -189,4 +189,14 @@ public class SpanAssert extends AbstractAssert<SpanAssert, Span> {
}
return this;
}
public SpanAssert isNotExportable() {
isNotNull();
if (this.actual.isExportable()) {
String message = "The span is NOT supposed to be exportable but it is!";
log.error(message);
failWithMessage(message);
}
return this;
}
}

View File

@@ -0,0 +1,142 @@
/*
* Copyright 2013-2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.sleuth.instrument.messaging;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import org.junit.Test;
import org.springframework.cloud.sleuth.Span;
import org.springframework.cloud.sleuth.SpanTextMap;
import static org.springframework.cloud.sleuth.assertions.SleuthAssertions.then;
/**
* @author Marcin Grzejszczak
*/
public class HeaderBasedMessagingExtractorTests {
@Test
public void overridesTheSampleFlagWithSpanFlagForSampledScenario() {
HeaderBasedMessagingExtractor extractor = new HeaderBasedMessagingExtractor();
SpanTextMap spanTextMap = spanTextMap();
spanTextMap.put(TraceMessageHeaders.SPAN_ID_NAME, Span.idToHex(10L));
spanTextMap.put(TraceMessageHeaders.TRACE_ID_NAME, Span.idToHex(20L));
spanTextMap.put(TraceMessageHeaders.SAMPLED_NAME, "0");
spanTextMap.put(TraceMessageHeaders.SPAN_FLAGS_NAME, "1");
Span span = extractor.joinTrace(spanTextMap);
then(span).isExportable();
}
@Test
public void doesNotOverrideTheSampleHeaderWithSpanFlagWhenTheSpanFlagIsNot1() {
HeaderBasedMessagingExtractor extractor = new HeaderBasedMessagingExtractor();
SpanTextMap spanTextMap = spanTextMap();
spanTextMap.put(TraceMessageHeaders.SPAN_ID_NAME, Span.idToHex(10L));
spanTextMap.put(TraceMessageHeaders.TRACE_ID_NAME, Span.idToHex(20L));
spanTextMap.put(TraceMessageHeaders.SAMPLED_NAME, "1");
spanTextMap.put(TraceMessageHeaders.SPAN_FLAGS_NAME, "0");
Span span = extractor.joinTrace(spanTextMap);
then(span).isExportable();
}
@Test
public void samplesASpanWhenSampledFlagIsSetTo1() {
HeaderBasedMessagingExtractor extractor = new HeaderBasedMessagingExtractor();
SpanTextMap spanTextMap = spanTextMap();
spanTextMap.put(TraceMessageHeaders.SPAN_ID_NAME, Span.idToHex(10L));
spanTextMap.put(TraceMessageHeaders.TRACE_ID_NAME, Span.idToHex(20L));
spanTextMap.put(TraceMessageHeaders.SAMPLED_NAME, "1");
Span span = extractor.joinTrace(spanTextMap);
then(span).isExportable();
}
@Test
public void doesNotSampleASpanWhenSampledFlagIsSetTo0() {
HeaderBasedMessagingExtractor extractor = new HeaderBasedMessagingExtractor();
SpanTextMap spanTextMap = spanTextMap();
spanTextMap.put(TraceMessageHeaders.SPAN_ID_NAME, Span.idToHex(10L));
spanTextMap.put(TraceMessageHeaders.TRACE_ID_NAME, Span.idToHex(20L));
spanTextMap.put(TraceMessageHeaders.SAMPLED_NAME, "0");
Span span = extractor.joinTrace(spanTextMap);
then(span).isNotExportable();
}
@Test
public void samplesWhenDebugFlagIsSetTo1RegardlessOfTraceAndSpanId() {
HeaderBasedMessagingExtractor extractor = new HeaderBasedMessagingExtractor();
SpanTextMap spanTextMap = spanTextMap();
spanTextMap.put(TraceMessageHeaders.SPAN_FLAGS_NAME, "1");
Span span = extractor.joinTrace(spanTextMap);
then(span).isExportable();
then(span.traceIdString()).isNotEmpty();
then(span.getSpanId()).isNotNull();
}
@Test
public void samplesWhenDebugFlagIsSetTo1AndOnlySpanIdIsSet() {
HeaderBasedMessagingExtractor extractor = new HeaderBasedMessagingExtractor();
SpanTextMap spanTextMap = spanTextMap();
spanTextMap.put(TraceMessageHeaders.SPAN_FLAGS_NAME, "1");
spanTextMap.put(TraceMessageHeaders.SPAN_ID_NAME, Span.idToHex(10L));
Span span = extractor.joinTrace(spanTextMap);
then(span).isExportable();
then(span.traceIdString()).isNotEmpty();
then(span.getSpanId()).isEqualTo(10L);
}
@Test
public void samplesWhenDebugFlagIsSetTo1AndOnlyTraceIdIsSet() {
HeaderBasedMessagingExtractor extractor = new HeaderBasedMessagingExtractor();
SpanTextMap spanTextMap = spanTextMap();
spanTextMap.put(TraceMessageHeaders.SPAN_FLAGS_NAME, "1");
spanTextMap.put(TraceMessageHeaders.TRACE_ID_NAME, Span.idToHex(10L));
Span span = extractor.joinTrace(spanTextMap);
then(span).isExportable();
then(span.getTraceId()).isEqualTo(10L);
then(span.getSpanId()).isEqualTo(10L);
}
private SpanTextMap spanTextMap() {
return new SpanTextMap() {
private final Map<String, String> map = new HashMap<>();
@Override public Iterator<Map.Entry<String, String>> iterator() {
return this.map.entrySet().iterator();
}
@Override public void put(String key, String value) {
this.map.put(key, value);
}
};
}
}

View File

@@ -382,6 +382,74 @@ public class TraceFilterTests {
then(this.response.getStatus()).isEqualTo(HttpStatus.OK.value());
}
@Test
public void samplesASpanRegardlessOfTheSamplerWhenXB3FlagsIsPresentAndSetTo1() throws Exception {
this.request = builder()
.header(Span.SPAN_FLAGS, 1)
.buildRequest(new MockServletContext());
this.sampler = new NeverSampler();
TraceFilter filter = new TraceFilter(this.tracer, this.traceKeys, this.spanReporter,
this.spanExtractor, this.httpTraceKeysInjector);
filter.doFilter(this.request, this.response, this.filterChain);
then(new ListOfSpans(this.spanReporter.getSpans())).allSpansAreExportable();
then(TestSpanContextHolder.getCurrentSpan()).isNull();
then(ExceptionUtils.getLastException()).isNull();
}
@Test
public void doesNotOverrideTheSampledFlagWhenXB3FlagIsSetToOtherValueThan1() throws Exception {
this.request = builder()
.header(Span.SPAN_FLAGS, 0)
.buildRequest(new MockServletContext());
this.sampler = new AlwaysSampler();
TraceFilter filter = new TraceFilter(this.tracer, this.traceKeys, this.spanReporter,
this.spanExtractor, this.httpTraceKeysInjector);
filter.doFilter(this.request, this.response, this.filterChain);
then(new ListOfSpans(this.spanReporter.getSpans())).allSpansAreExportable();
then(TestSpanContextHolder.getCurrentSpan()).isNull();
then(ExceptionUtils.getLastException()).isNull();
}
@Test
public void samplesWhenDebugFlagIsSetTo1AndOnlySpanIdIsSet() throws Exception {
this.request = builder()
.header(Span.SPAN_FLAGS, 1)
.header(Span.SPAN_ID_NAME, 10L)
.buildRequest(new MockServletContext());
this.sampler = new NeverSampler();
TraceFilter filter = new TraceFilter(this.tracer, this.traceKeys, this.spanReporter,
this.spanExtractor, this.httpTraceKeysInjector);
filter.doFilter(this.request, this.response, this.filterChain);
then(new ListOfSpans(this.spanReporter.getSpans()))
.allSpansAreExportable().hasSize(2).hasASpanWithSpanId(Span.hexToId("10"));
then(TestSpanContextHolder.getCurrentSpan()).isNull();
then(ExceptionUtils.getLastException()).isNull();
}
@Test
public void samplesWhenDebugFlagIsSetTo1AndTraceIdIsAlsoSet() throws Exception {
this.request = builder()
.header(Span.SPAN_FLAGS, 1)
.header(Span.TRACE_ID_NAME, 10L)
.buildRequest(new MockServletContext());
this.sampler = new NeverSampler();
TraceFilter filter = new TraceFilter(this.tracer, this.traceKeys, this.spanReporter,
this.spanExtractor, this.httpTraceKeysInjector);
filter.doFilter(this.request, this.response, this.filterChain);
then(new ListOfSpans(this.spanReporter.getSpans()))
.allSpansAreExportable().allSpansHaveTraceId(Span.hexToId("10"));
then(TestSpanContextHolder.getCurrentSpan()).isNull();
then(ExceptionUtils.getLastException()).isNull();
}
public void verifyParentSpanHttpTags() {
verifyParentSpanHttpTags(HttpStatus.OK);
}

View File

@@ -72,6 +72,7 @@ final class ConvertToZipkinSpanList {
*/
// VisibleForTesting
static zipkin.Span convert(Span span, Host host) {
//TODO: Consider adding support for the debug flag (related to #496)
Builder zipkinSpan = zipkin.Span.builder();
Endpoint ep = Endpoint.builder()

View File

@@ -87,6 +87,7 @@ public class ZipkinSpanListener implements SpanReporter {
*/
// Visible for testing
zipkin.Span convert(Span span) {
//TODO: Consider adding support for the debug flag (related to #496)
zipkin.Span.Builder zipkinSpan = zipkin.Span.builder();
Endpoint endpoint = this.endpointLocator.local();