WIP

pom.xml

@@ -71,6 +71,7 @@
 		<spring-cloud-netflix.version>3.0.2-SNAPSHOT</spring-cloud-netflix.version>
 		<spring-cloud-openfeign.version>3.0.2-SNAPSHOT</spring-cloud-openfeign.version>
 		<brave.version>5.13.2</brave.version>
+		<wavefront-spring-boot.version>2.1.1-SNAPSHOT</wavefront-spring-boot.version>
 		<opentracing.version>0.32.0</opentracing.version>
 		<spring-security-boot-autoconfigure.version>2.3.4.RELEASE</spring-security-boot-autoconfigure.version>
 		<disable.nohttp.checks>false</disable.nohttp.checks>
@@ -228,6 +229,13 @@
 				<type>pom</type>
 				<scope>import</scope>
 			</dependency>
+			<dependency>
+				<groupId>com.wavefront</groupId>
+				<artifactId>wavefront-spring-boot-bom</artifactId>
+				<version>${wavefront-spring-boot.version}</version>
+				<type>pom</type>
+				<scope>import</scope>
+			</dependency>
 			<dependency>
 				<groupId>org.springframework.security.oauth.boot</groupId>
 				<artifactId>spring-security-oauth2-autoconfigure</artifactId>

spring-cloud-sleuth-autoconfigure/pom.xml

@@ -365,6 +365,34 @@
 			</exclusions>
 		</dependency>
 
+		<!-- Wavefront -->
+		<dependency>
+			<groupId>com.wavefront</groupId>
+			<artifactId>wavefront-opentracing-sdk-java</artifactId>
+			<optional>true</optional>
+		</dependency>
+		<dependency>
+			<groupId>com.wavefront</groupId>
+			<artifactId>wavefront-spring-boot</artifactId>
+			<optional>true</optional>
+		</dependency>
+		<dependency>
+			<groupId>com.wavefront</groupId>
+			<artifactId>wavefront-runtime-sdk-jvm</artifactId>
+			<optional>true</optional>
+		</dependency>
+		<dependency>
+			<groupId>com.wavefront</groupId>
+			<artifactId>wavefront-sdk-java</artifactId>
+			<optional>true</optional>
+		</dependency>
+		<dependency>
+			<groupId>io.micrometer</groupId>
+			<artifactId>micrometer-registry-wavefront</artifactId>
+			<optional>true</optional>
+		</dependency>
+
+		<!-- Tests -->
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-starter-test</artifactId>

spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontProperties.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2013-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.sleuth.autoconfig.wavefront;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+/**
+ * Advanced configuration properties for Wavefront.
+ *
+ * @author Stephane Nicoll
+ */
+@ConfigurationProperties("wavefront.tracing")
+public class WavefrontProperties {
+
+	/**
+	 * Enables Wavefront tracing.
+	 */
+	private boolean enabled;
+
+	/**
+	 * Tags that should be associated with RED metrics. If the span has any of the
+	 * specified tags, then those get reported to generated RED metrics.
+	 */
+	private Set<String> redMetricsCustomTagKeys = new HashSet<>();
+
+	public Set<String> getRedMetricsCustomTagKeys() {
+		return this.redMetricsCustomTagKeys;
+	}
+
+	public void setRedMetricsCustomTagKeys(Set<String> redMetricsCustomTagKeys) {
+		this.redMetricsCustomTagKeys = redMetricsCustomTagKeys;
+	}
+
+	public boolean isEnabled() {
+		return this.enabled;
+	}
+
+	public void setEnabled(boolean enabled) {
+		this.enabled = enabled;
+	}
+
+}

spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthAutoConfiguration.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2013-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.sleuth.autoconfig.wavefront;
+
+import brave.Tracer;
+import brave.TracingCustomizer;
+import brave.handler.SpanHandler;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.wavefront.sdk.common.application.ApplicationTags;
+import com.wavefront.spring.autoconfigure.WavefrontAutoConfiguration;
+import com.wavefront.spring.autoconfigure.WavefrontTracerBla;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.wavefront.WavefrontConfig;
+
+import org.springframework.boot.actuate.autoconfigure.metrics.export.wavefront.WavefrontMetricsExportAutoConfiguration;
+import org.springframework.boot.autoconfigure.AutoConfigureAfter;
+import org.springframework.boot.autoconfigure.AutoConfigureBefore;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.cloud.sleuth.SpanNamer;
+import org.springframework.cloud.sleuth.autoconfig.brave.BraveAutoConfiguration;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * Configuration for Wavefront tracing using Spring Cloud Sleuth.
+ *
+ * @author Adrian Cole
+ * @author Stephane Nicoll
+ * @since 3.1.0
+ */
+@Configuration(proxyBeanMethods = false)
+@ConditionalOnClass({ SpanNamer.class, MeterRegistry.class, WavefrontConfig.class, WavefrontSender.class })
+@ConditionalOnBean(WavefrontSender.class)
+@AutoConfigureBefore({ BraveAutoConfiguration.class, WavefrontAutoConfiguration.class })
+@AutoConfigureAfter(WavefrontMetricsExportAutoConfiguration.class)
+@EnableConfigurationProperties(WavefrontProperties.class)
+@ConditionalOnProperty(value = "wavefront.tracing.enabled", matchIfMissing = true)
+public class WavefrontSleuthAutoConfiguration {
+
+	static final String BEAN_NAME = "wavefrontTracingCustomizer";
+
+	@Bean
+	@ConditionalOnBean({ MeterRegistry.class, WavefrontConfig.class, WavefrontSender.class })
+	WavefrontSleuthSpanHandler wavefrontSleuthSpanHandler(MeterRegistry meterRegistry, WavefrontSender wavefrontSender,
+			ApplicationTags applicationTags, WavefrontConfig wavefrontConfig, WavefrontProperties wavefrontProperties) {
+		return new WavefrontSleuthSpanHandler(
+				// https://github.com/wavefrontHQ/wavefront-opentracing-sdk-java/blob/f1f08d8daf7b692b9b61dcd5bc24ca6befa8e710/src/main/java/com/wavefront/opentracing/reporting/WavefrontSpanReporter.java#L54
+				50000, // TODO: maxQueueSize should be a property, ya?
+				wavefrontSender, meterRegistry, wavefrontConfig.source(), applicationTags, wavefrontProperties);
+	}
+
+	@Configuration(proxyBeanMethods = false)
+	@ConditionalOnClass({ Tracer.class, TracingCustomizer.class, SpanHandler.class })
+	static class BraveCustomizerConfiguration {
+
+		@Bean(BEAN_NAME)
+		@ConditionalOnMissingBean(name = BEAN_NAME)
+		@ConditionalOnBean({ MeterRegistry.class, WavefrontConfig.class, WavefrontSender.class })
+		TracingCustomizer wavefrontTracingCustomizer(WavefrontSleuthSpanHandler spanHandler) {
+			return t -> t.traceId128Bit(true).supportsJoin(false)
+					.addSpanHandler(new WavefrontSleuthBraveSpanHandler(spanHandler));
+		}
+
+		@Bean
+		WavefrontTracerBla wavefrontTracerBla() {
+			return new WavefrontTracerBla() {
+			};
+		}
+
+	}
+
+}

spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthBraveSpanHandler.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2013-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.sleuth.autoconfig.wavefront;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import brave.handler.MutableSpan;
+import brave.handler.SpanHandler;
+import brave.propagation.TraceContext;
+
+import org.springframework.cloud.sleuth.brave.bridge.BraveFinishedSpan;
+import org.springframework.cloud.sleuth.brave.bridge.BraveTraceContext;
+
+class WavefrontSleuthBraveSpanHandler extends SpanHandler implements Runnable, Closeable {
+
+	final WavefrontSleuthSpanHandler spanHandler;
+
+	WavefrontSleuthBraveSpanHandler(WavefrontSleuthSpanHandler spanHandler) {
+		this.spanHandler = spanHandler;
+	}
+
+	@Override
+	public boolean end(TraceContext context, MutableSpan span, Cause cause) {
+		return spanHandler.end(BraveTraceContext.fromBrave(context), BraveFinishedSpan.fromBrave(span));
+	}
+
+	@Override
+	public void close() throws IOException {
+		this.spanHandler.close();
+	}
+
+	@Override
+	public void run() {
+		this.spanHandler.run();
+	}
+
+}

spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthSpanHandler.java

@@ -0,0 +1,467 @@
+/*
+ * Copyright 2013-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.sleuth.autoconfig.wavefront;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import com.wavefront.internal.reporter.WavefrontInternalReporter;
+import com.wavefront.java_sdk.com.google.common.collect.Iterators;
+import com.wavefront.java_sdk.com.google.common.collect.Sets;
+import com.wavefront.sdk.common.NamedThreadFactory;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.wavefront.sdk.common.application.ApplicationTags;
+import com.wavefront.sdk.entities.tracing.SpanLog;
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.MeterRegistry;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.springframework.cloud.sleuth.TraceContext;
+import org.springframework.cloud.sleuth.exporter.FinishedSpan;
+import org.springframework.util.StringUtils;
+
+import static com.wavefront.internal.SpanDerivedMetricsUtils.TRACING_DERIVED_PREFIX;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportHeartbeats;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportWavefrontGeneratedData;
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.COMPONENT_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.DEBUG_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.ERROR_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.NULL_TAG_VAL;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SOURCE_KEY;
+import static com.wavefront.sdk.common.Constants.SPAN_LOG_KEY;
+
+/**
+ * This converts a span recorded by Brave and invokes {@link WavefrontSender#sendSpan}.
+ *
+ * <p>
+ * This uses a combination of conversion approaches from Wavefront projects:
+ * <ul>
+ * <li>https://github.com/wavefrontHQ/wavefront-opentracing-sdk-java</li>
+ * <li>https://github.com/wavefrontHQ/wavefront-proxy</li>
+ * </ul>
+ *
+ * <p>
+ * On conflict, we make a comment and prefer wavefront-opentracing-sdk-java. The rationale
+ * is wavefront-opentracing-sdk-java uses the same {@link WavefrontSender#sendSpan}
+ * library, so it is easier to reason with. This policy can be revisited by future
+ * maintainers.
+ *
+ * <p>
+ * <em>Note:</em>UUID conversions follow the same conventions used in practice in
+ * Wavefront. Ex.
+ * https://github.com/wavefrontHQ/wavefront-opentracing-sdk-java/blob/6babf2ff95daa37452e1e8c35ae54b58b6abb50f/src/main/java/com/wavefront/opentracing/propagation/JaegerWavefrontPropagator.java#L191-L204
+ * While in practice this is not a problem, it is worth mentioning that this convention
+ * will only only result in RFC 4122 timestamp (version 1) format by accident. In other
+ * words, don't call {@link UUID#timestamp()} on UUIDs converted here, or in other
+ * Wavefront code, as it might throw.
+ *
+ * @since 3.1.0
+ */
+public final class WavefrontSleuthSpanHandler implements Runnable, Closeable {
+
+	private static final Log LOG = LogFactory.getLog(WavefrontSleuthSpanHandler.class);
+
+	// https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L114-L114
+	private static final String DEFAULT_SPAN_NAME = "defaultOperation";
+
+	private final static String DEFAULT_SOURCE = "wavefront-spring-boot";
+
+	private final static String WAVEFRONT_GENERATED_COMPONENT = "wavefront-generated";
+
+	private static final int LONG_BYTES = Long.SIZE / Byte.SIZE;
+
+	private static final int BYTE_BASE16 = 2;
+
+	private static final int LONG_BASE16 = BYTE_BASE16 * LONG_BYTES;
+
+	private static final int TRACE_ID_HEX_SIZE = 2 * LONG_BASE16;
+
+	private static final String ALPHABET = "0123456789abcdef";
+
+	private static final int ASCII_CHARACTERS = 128;
+
+	private static final byte[] DECODING = buildDecodingArray();
+
+	final LinkedBlockingQueue<Pair<TraceContext, FinishedSpan>> spanBuffer;
+
+	final WavefrontSender wavefrontSender;
+
+	final WavefrontInternalReporter wfInternalReporter;
+
+	final Set<String> traceDerivedCustomTagKeys;
+
+	final Counter spansDropped;
+
+	final Counter spansReceived;
+
+	final Counter reportErrors;
+
+	final Thread sendingThread;
+
+	private volatile boolean stop = false;
+
+	private final Set<Pair<Map<String, String>, String>> discoveredHeartbeatMetrics;
+
+	private final ScheduledExecutorService heartbeatMetricsScheduledExecutorService;
+
+	final String source;
+
+	final List<Pair<String, String>> defaultTags;
+
+	final Set<String> defaultTagKeys;
+
+	final ApplicationTags applicationTags;
+
+	WavefrontSleuthSpanHandler(int maxQueueSize, WavefrontSender wavefrontSender, MeterRegistry meterRegistry,
+			String source, ApplicationTags applicationTags, WavefrontProperties wavefrontProperties) {
+		this.wavefrontSender = wavefrontSender;
+		this.applicationTags = applicationTags;
+		this.discoveredHeartbeatMetrics = Sets.newConcurrentHashSet();
+
+		this.heartbeatMetricsScheduledExecutorService = Executors.newScheduledThreadPool(1,
+				new NamedThreadFactory("sleuth-heart-beater").setDaemon(true));
+
+		// Emit Heartbeats Metrics every 1 min.
+		heartbeatMetricsScheduledExecutorService.scheduleAtFixedRate(() -> {
+			try {
+				reportHeartbeats(wavefrontSender, discoveredHeartbeatMetrics, WAVEFRONT_GENERATED_COMPONENT);
+			}
+			catch (IOException e) {
+				LOG.warn("Cannot report heartbeat metric to wavefront");
+			}
+		}, 1, 60, TimeUnit.SECONDS);
+
+		this.traceDerivedCustomTagKeys = new HashSet<>(wavefrontProperties.getRedMetricsCustomTagKeys());
+
+		// Start the reporter
+		wfInternalReporter = new WavefrontInternalReporter.Builder().prefixedWith(TRACING_DERIVED_PREFIX)
+				.withSource(DEFAULT_SOURCE).reportMinuteDistribution().build(wavefrontSender);
+		wfInternalReporter.start(1, TimeUnit.MINUTES);
+
+		this.source = source;
+		this.defaultTags = createDefaultTags(applicationTags);
+		this.defaultTagKeys = defaultTags.stream().map(p -> p._1).collect(Collectors.toSet());
+		this.defaultTagKeys.add(SOURCE_KEY);
+
+		this.spanBuffer = new LinkedBlockingQueue<>(maxQueueSize);
+
+		// init internal metrics
+		meterRegistry.gauge("reporter.queue.size", spanBuffer, sb -> (double) sb.size());
+		meterRegistry.gauge("reporter.queue.remaining_capacity", spanBuffer, sb -> (double) sb.remainingCapacity());
+		this.spansReceived = meterRegistry.counter("reporter.spans.received");
+		this.spansDropped = meterRegistry.counter("reporter.spans.dropped");
+		this.reportErrors = meterRegistry.counter("reporter.errors");
+
+		this.sendingThread = new Thread(this, "wavefrontSpanReporter");
+		this.sendingThread.setDaemon(true);
+		this.sendingThread.start();
+	}
+
+	// Exact same behavior as WavefrontSpanReporter
+	// https://github.com/wavefrontHQ/wavefront-opentracing-sdk-java/blob/f1f08d8daf7b692b9b61dcd5bc24ca6befa8e710/src/main/java/com/wavefront/opentracing/reporting/WavefrontSpanReporter.java#L163-L179
+	public boolean end(TraceContext context, FinishedSpan span) {
+		spansReceived.increment();
+		if (!spanBuffer.offer(Pair.of(context, span))) {
+			spansDropped.increment();
+			if (LOG.isWarnEnabled()) {
+				LOG.warn("Buffer full, dropping span: " + span);
+				LOG.warn("Total spans dropped: " + spansDropped.count());
+			}
+		}
+		return true; // regardless of error, other handlers should run
+	}
+
+	List<Pair<String, String>> getDefaultTags() {
+		return Collections.unmodifiableList(this.defaultTags);
+	}
+
+	private String padLeftWithZeros(String string, int length) {
+		if (string.length() >= length) {
+			return string;
+		}
+		else {
+			StringBuilder sb = new StringBuilder(length);
+			for (int i = string.length(); i < length; i++) {
+				sb.append('0');
+			}
+
+			return sb.append(string).toString();
+		}
+	}
+
+	private void send(TraceContext context, FinishedSpan span) {
+		String traceIdString = padLeftWithZeros(context.traceId(), TRACE_ID_HEX_SIZE);
+		String traceIdHigh = traceIdString.substring(0, traceIdString.length() / 2);
+		String traceIdLow = traceIdString.substring(traceIdString.length() / 2);
+		UUID traceId = new UUID(longFromBase16String(traceIdHigh), longFromBase16String(traceIdLow));
+		UUID spanId = new UUID(0L, longFromBase16String(context.spanId()));
+
+		// NOTE: wavefront-opentracing-sdk-java and wavefront-proxy differ, but we prefer
+		// the former.
+		// https://github.com/wavefrontHQ/wavefront-opentracing-sdk-java/blob/f1f08d8daf7b692b9b61dcd5bc24ca6befa8e710/src/main/java/com/wavefront/opentracing/reporting/WavefrontSpanReporter.java#L187-L190
+		// https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L248-L252
+		List<UUID> parents = null;
+		String parentId = context.parentId();
+		if (StringUtils.hasText(parentId) && longFromBase16String(parentId) != 0L) {
+			parents = Collections.singletonList(new UUID(0L, longFromBase16String(parentId)));
+		}
+		List<UUID> followsFrom = null;
+
+		// https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L344-L345
+		String name = span.getName();
+		if (name == null) {
+			name = DEFAULT_SPAN_NAME;
+		}
+
+		// Start and duration become 0L if unset. Any positive duration rounds up to 1
+		// millis.
+		long startMillis = span.getStartTimestamp() / 1000L;
+		long finishMillis = span.getEndTimestamp() / 1000L;
+		long durationMicros = span.getEndTimestamp() - span.getStartTimestamp();
+		long durationMillis = startMillis != 0 && finishMillis != 0L ? Math.max(finishMillis - startMillis, 1L) : 0L;
+
+		List<SpanLog> spanLogs = convertAnnotationsToSpanLogs(span);
+		TagList tags = new TagList(defaultTagKeys, defaultTags, span);
+
+		try {
+			wavefrontSender.sendSpan(name, startMillis, durationMillis, source, traceId, spanId, parents, followsFrom,
+					tags, spanLogs);
+		}
+		catch (IOException | RuntimeException t) {
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("error sending span " + context, t);
+			}
+		}
+
+		// report stats irrespective of span sampling.
+		if (wfInternalReporter != null) {
+			// report converted metrics/histograms from the span
+			try {
+				discoveredHeartbeatMetrics.add(reportWavefrontGeneratedData(wfInternalReporter, name,
+						applicationTags.getApplication(), applicationTags.getService(),
+						applicationTags.getCluster() == null ? NULL_TAG_VAL : applicationTags.getCluster(),
+						applicationTags.getShard() == null ? NULL_TAG_VAL : applicationTags.getShard(), source,
+						tags.componentTagValue, tags.isError, durationMicros, traceDerivedCustomTagKeys, tags));
+			}
+			catch (RuntimeException t) {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("error sending span RED metrics " + context, t);
+				}
+			}
+		}
+	}
+
+	private static byte[] buildDecodingArray() {
+		byte[] decoding = new byte[ASCII_CHARACTERS];
+		Arrays.fill(decoding, (byte) -1);
+		for (int i = 0; i < ALPHABET.length(); i++) {
+			char c = ALPHABET.charAt(i);
+			decoding[c] = (byte) i;
+		}
+		return decoding;
+	}
+
+	/**
+	 * Returns the {@code long} value whose base16 representation is stored in the first
+	 * 16 chars of {@code chars} starting from the {@code offset}.
+	 * @param chars the base16 representation of the {@code long}.
+	 */
+	private static long longFromBase16String(CharSequence chars) {
+		int offset = 0;
+		return (decodeByte(chars.charAt(offset), chars.charAt(offset + 1)) & 0xFFL) << 56
+				| (decodeByte(chars.charAt(offset + 2), chars.charAt(offset + 3)) & 0xFFL) << 48
+				| (decodeByte(chars.charAt(offset + 4), chars.charAt(offset + 5)) & 0xFFL) << 40
+				| (decodeByte(chars.charAt(offset + 6), chars.charAt(offset + 7)) & 0xFFL) << 32
+				| (decodeByte(chars.charAt(offset + 8), chars.charAt(offset + 9)) & 0xFFL) << 24
+				| (decodeByte(chars.charAt(offset + 10), chars.charAt(offset + 11)) & 0xFFL) << 16
+				| (decodeByte(chars.charAt(offset + 12), chars.charAt(offset + 13)) & 0xFFL) << 8
+				| (decodeByte(chars.charAt(offset + 14), chars.charAt(offset + 15)) & 0xFFL);
+	}
+
+	private static byte decodeByte(char hi, char lo) {
+		int decoded = DECODING[hi] << 4 | DECODING[lo];
+		return (byte) decoded;
+	}
+
+	@Override
+	public void run() {
+		while (!stop) {
+			try {
+				Pair<TraceContext, FinishedSpan> contextAndSpan = spanBuffer.take();
+				send(contextAndSpan._1, contextAndSpan._2);
+			}
+			catch (InterruptedException ex) {
+				if (LOG.isInfoEnabled()) {
+					LOG.info("reporting thread interrupted");
+				}
+			}
+			catch (Throwable ex) {
+				LOG.warn("Error processing buffer", ex);
+			}
+		}
+	}
+
+	@Override
+	public void close() {
+		stop = true;
+		try {
+			// wait for 5 secs max
+			sendingThread.join(5000);
+			heartbeatMetricsScheduledExecutorService.shutdownNow();
+		}
+		catch (InterruptedException ex) {
+			// no-op
+		}
+	}
+
+	// https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L397-L402
+	static List<SpanLog> convertAnnotationsToSpanLogs(FinishedSpan span) {
+		int annotationCount = span.getEvents().size();
+		if (annotationCount == 0) {
+			return Collections.emptyList();
+		}
+		List<SpanLog> spanLogs = new ArrayList<>(annotationCount);
+		for (int i = 0; i < annotationCount; i++) {
+			Map.Entry<Long, String> entry = Iterators.get(span.getEvents().iterator(), i);
+			long epochMicros = entry.getKey();
+			String value = entry.getValue();
+			spanLogs.add(new SpanLog(epochMicros, Collections.singletonMap("annotation", value)));
+		}
+		return spanLogs;
+	}
+
+	// https://github.com/wavefrontHQ/wavefront-opentracing-sdk-java/blob/f1f08d8daf7b692b9b61dcd5bc24ca6befa8e710/src/main/java/com/wavefront/opentracing/WavefrontTracer.java#L275-L280
+	static List<Pair<String, String>> createDefaultTags(ApplicationTags applicationTags) {
+		List<Pair<String, String>> result = new ArrayList<>();
+		result.add(Pair.of(APPLICATION_TAG_KEY, applicationTags.getApplication()));
+		result.add(Pair.of(SERVICE_TAG_KEY, applicationTags.getService()));
+		result.add(Pair.of(CLUSTER_TAG_KEY,
+				applicationTags.getCluster() == null ? NULL_TAG_VAL : applicationTags.getCluster()));
+		result.add(
+				Pair.of(SHARD_TAG_KEY, applicationTags.getShard() == null ? NULL_TAG_VAL : applicationTags.getShard()));
+		if (applicationTags.getCustomTags() != null) {
+			applicationTags.getCustomTags().forEach((k, v) -> result.add(Pair.of(k, v)));
+		}
+		return result;
+	}
+
+	/**
+	 * Extracted for test isolation and as parsing otherwise implies multiple-returns or
+	 * scanning later.
+	 *
+	 * <p>
+	 * Ex. {@code SpanDerivedMetricsUtils#reportWavefrontGeneratedData} needs tags
+	 * separately from the component tag and error status.
+	 */
+	static final class TagList extends ArrayList<Pair<String, String>> {
+
+		String componentTagValue = NULL_TAG_VAL;
+
+		boolean isError; // See explanation here:
+
+		// https://github.com/openzipkin/brave/pull/1221
+
+		TagList(Set<String> defaultTagKeys, List<Pair<String, String>> defaultTags, FinishedSpan span) {
+			super(defaultTags.size() + span.getTags().size());
+			// TODO: OTel doesn't have a notion of debug
+			boolean debug = false;
+			boolean hasAnnotations = span.getEvents().size() > 0;
+			isError = span.getError() != null;
+
+			int tagCount = span.getTags().size();
+			addAll(defaultTags);
+			for (int i = 0; i < tagCount; i++) {
+				String tagKey = Iterators.get(span.getTags().keySet().iterator(), i);
+				String tagValue = Iterators.get(span.getTags().values().iterator(), i);
+				String key = tagKey;
+				String value = tagValue;
+				String lcKey = key.toLowerCase(Locale.ROOT);
+				if (lcKey.equals(ERROR_TAG_KEY)) {
+					isError = true;
+					continue; // We later replace whatever the potentially empty value was
+					// with "true"
+				}
+				if (value.isEmpty()) {
+					continue;
+				}
+				if (defaultTagKeys.contains(lcKey)) {
+					continue;
+				}
+				if (lcKey.equals(DEBUG_TAG_KEY)) {
+					debug = true; // This tag is set out-of-band
+					continue;
+				}
+				if (lcKey.equals(COMPONENT_TAG_KEY)) {
+					componentTagValue = value;
+				}
+				add(Pair.of(key, value));
+			}
+
+			// Check for span.error() for uncaught exception in request mapping and add it
+			// to Wavefront span tag
+			if (isError) {
+				add(Pair.of("error", "true"));
+			}
+
+			// https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L300-L303
+			if (debug) {
+				add(Pair.of(DEBUG_TAG_KEY, "true"));
+			}
+
+			// https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L254-L266
+			if (span.getKind() != null) {
+				String kind = span.getKind().toString().toLowerCase();
+				add(Pair.of("span.kind", kind));
+				if (hasAnnotations) {
+					add(Pair.of("_spanSecondaryId", kind));
+				}
+			}
+
+			// https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L329-L332
+			if (hasAnnotations) {
+				add(Pair.of(SPAN_LOG_KEY, "true"));
+			}
+
+			// https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L324-L327
+			if (span.getLocalIp() != null) {
+				add(Pair.of("ipv4", span.getLocalIp())); // NOTE: this could be IPv6!!
+			}
+		}
+
+	}
+
+}

spring-cloud-sleuth-autoconfigure/src/main/resources/META-INF/spring.factories

@@ -25,7 +25,8 @@ org.springframework.cloud.sleuth.autoconfig.brave.instrument.messaging.BraveMess
 org.springframework.cloud.sleuth.autoconfig.brave.instrument.opentracing.BraveOpentracingAutoConfiguration,\
 org.springframework.cloud.sleuth.autoconfig.brave.instrument.redis.BraveRedisAutoConfiguration,\
 org.springframework.cloud.sleuth.autoconfig.brave.instrument.mongodb.BraveMongoDbAutoConfiguration,\
-org.springframework.cloud.sleuth.autoconfig.zipkin2.ZipkinAutoConfiguration
+org.springframework.cloud.sleuth.autoconfig.zipkin2.ZipkinAutoConfiguration,\
+org.springframework.cloud.sleuth.autoconfig.wavefront.WavefrontSleuthAutoConfiguration
 # Environment Post Processor
 org.springframework.boot.env.EnvironmentPostProcessor=\
 org.springframework.cloud.sleuth.autoconfig.TraceEnvironmentPostProcessor,\

spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/NoOpWavefrontSender.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2013-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.sleuth.autoconfig;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.wavefront.sdk.entities.histograms.HistogramGranularity;
+import com.wavefront.sdk.entities.tracing.SpanLog;
+
+/**
+ * A noop implementation. Does nothing.
+ *
+ * @author Marcin Grzejszczak
+ * @since 3.0.0
+ */
+public class NoOpWavefrontSender implements WavefrontSender {
+
+	@Override
+	public String getClientId() {
+		return null;
+	}
+
+	@Override
+	public void flush() {
+
+	}
+
+	@Override
+	public int getFailureCount() {
+		return 0;
+	}
+
+	@Override
+	public void sendDistribution(String name, List<Pair<Double, Integer>> centroids,
+			Set<HistogramGranularity> histogramGranularities, Long timestamp, String source, Map<String, String> tags) {
+
+	}
+
+	@Override
+	public void sendMetric(String name, double value, Long timestamp, String source, Map<String, String> tags) {
+
+	}
+
+	@Override
+	public void sendFormattedMetric(String point) {
+
+	}
+
+	@Override
+	public void sendSpan(String name, long startMillis, long durationMillis, String source, UUID traceId, UUID spanId,
+			List<UUID> parents, List<UUID> followsFrom, List<Pair<String, String>> tags, List<SpanLog> spanLogs) {
+	}
+
+	@Override
+	public void close() {
+
+	}
+
+}

spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/TraceNoOpAutoConfiguration.java

@@ -16,6 +16,7 @@
 
 package org.springframework.cloud.sleuth.autoconfig;
 
+import org.springframework.boot.actuate.autoconfigure.metrics.export.wavefront.WavefrontMetricsExportAutoConfiguration;
 import org.springframework.boot.autoconfigure.AutoConfigureBefore;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -31,6 +32,7 @@ import org.springframework.cloud.sleuth.propagation.Propagator;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
 
 /**
  * {@link org.springframework.boot.autoconfigure.EnableAutoConfiguration
@@ -43,7 +45,7 @@ import org.springframework.context.annotation.Import;
  */
 @Configuration(proxyBeanMethods = false)
 @ConditionalOnProperty("spring.sleuth.noop.enabled")
-@AutoConfigureBefore(BraveAutoConfiguration.class)
+@AutoConfigureBefore({ BraveAutoConfiguration.class, WavefrontMetricsExportAutoConfiguration.class })
 @Import(TraceConfiguration.class)
 public class TraceNoOpAutoConfiguration {
 
@@ -68,6 +70,12 @@ public class TraceNoOpAutoConfiguration {
 		return new NoOpSpanCustomizer();
 	}
 
+	@Bean
+	@Primary
+	NoOpWavefrontSender noOpWavefrontSender() {
+		return new NoOpWavefrontSender();
+	}
+
 	@Configuration(proxyBeanMethods = false)
 	static class TraceHttpConfiguration {
 

spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontAutoConfigurationTests.java

@@ -0,0 +1,324 @@
+/*
+ * Copyright 2013-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.sleuth.autoconfig.wavefront;
+
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+import brave.Tracer;
+import brave.TracingCustomizer;
+import brave.handler.SpanHandler;
+import com.wavefront.sdk.appagent.jvm.reporter.WavefrontJvmReporter;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.wavefront.sdk.common.application.ApplicationTags;
+import com.wavefront.spring.autoconfigure.ApplicationTagsBuilderCustomizer;
+import com.wavefront.spring.autoconfigure.WavefrontAutoConfiguration;
+import io.micrometer.core.instrument.MeterRegistry;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.boot.actuate.autoconfigure.metrics.CompositeMeterRegistryAutoConfiguration;
+import org.springframework.boot.actuate.autoconfigure.metrics.MetricsAutoConfiguration;
+import org.springframework.boot.actuate.autoconfigure.metrics.export.simple.SimpleMetricsExportAutoConfiguration;
+import org.springframework.boot.actuate.autoconfigure.metrics.export.wavefront.WavefrontMetricsExportAutoConfiguration;
+import org.springframework.boot.autoconfigure.AutoConfigurations;
+import org.springframework.boot.test.context.FilteredClassLoader;
+import org.springframework.boot.test.context.assertj.AssertableApplicationContext;
+import org.springframework.boot.test.context.runner.AbstractApplicationContextRunner;
+import org.springframework.boot.test.context.runner.ApplicationContextRunner;
+import org.springframework.boot.test.context.runner.ContextConsumer;
+import org.springframework.cloud.sleuth.autoconfig.brave.BraveAutoConfiguration;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Tests for {@link WavefrontAutoConfiguration}.
+ *
+ * @author Stephane Nicoll
+ * @author Tommy Ludwig
+ */
+class WavefrontAutoConfigurationTests {
+
+	private final ApplicationContextRunner contextRunner = new ApplicationContextRunner().withConfiguration(
+			AutoConfigurations.of(WavefrontAutoConfiguration.class, WavefrontSleuthAutoConfiguration.class));
+
+	@Test
+	void applicationTagsIsConfiguredFromPropertiesWhenNoneExists() {
+		this.contextRunner
+				.withPropertyValues("wavefront.application.name=test-app", "wavefront.application.service=test-service")
+				.run((context) -> {
+					assertThat(context).hasSingleBean(ApplicationTags.class);
+					ApplicationTags tags = context.getBean(ApplicationTags.class);
+					assertThat(tags.getApplication()).isEqualTo("test-app");
+					assertThat(tags.getService()).isEqualTo("test-service");
+					assertThat(tags.getCluster()).isNull();
+					assertThat(tags.getShard()).isNull();
+					assertThat(tags.getCustomTags()).isEmpty();
+				});
+	}
+
+	@Test
+	void applicationTagsCanBeCustomized() {
+		this.contextRunner
+				.withPropertyValues("wavefront.application.name=test-app", "wavefront.application.service=test-service")
+				.withBean(ApplicationTagsBuilderCustomizer.class,
+						() -> (builder) -> builder.cluster("test-cluster").shard("test-shard"))
+				.run((context) -> {
+					assertThat(context).hasSingleBean(ApplicationTags.class);
+					ApplicationTags tags = context.getBean(ApplicationTags.class);
+					assertThat(tags.getApplication()).isEqualTo("test-app");
+					assertThat(tags.getService()).isEqualTo("test-service");
+					assertThat(tags.getCluster()).isEqualTo("test-cluster");
+					assertThat(tags.getShard()).isEqualTo("test-shard");
+					assertThat(tags.getCustomTags()).isEmpty();
+				});
+	}
+
+	@Test
+	void applicationTagsIsReusedWhenCustomInstanceExists() {
+		this.contextRunner
+				.withPropertyValues("wavefront.application.name=test-app", "wavefront.application.service=test-service")
+				.withBean(ApplicationTags.class,
+						() -> new ApplicationTags.Builder("another-app", "another-service").build())
+				.run((context) -> {
+					assertThat(context).hasSingleBean(ApplicationTags.class);
+					ApplicationTags tags = context.getBean(ApplicationTags.class);
+					assertThat(tags.getApplication()).isEqualTo("another-app");
+					assertThat(tags.getService()).isEqualTo("another-service");
+					assertThat(tags.getCluster()).isNull();
+					assertThat(tags.getShard()).isNull();
+					assertThat(tags.getCustomTags()).isEmpty();
+				});
+	}
+
+	@Test
+	void applicationTagsAreExportedToWavefrontRegistry() {
+		this.contextRunner
+				.withPropertyValues("wavefront.application.name=test-app", "wavefront.application.service=test-service")
+				.with(wavefrontMetrics(() -> mock(WavefrontSender.class))).run((context) -> {
+					MeterRegistry registry = context.getBean(MeterRegistry.class);
+					registry.counter("my.counter", "env", "qa");
+					assertThat(registry.find("my.counter").tags("env", "qa").tags("application", "test-app")
+							.tags("service", "test-service").counter()).isNotNull();
+				});
+	}
+
+	@Test
+	void applicationTagsWithFullInformationAreExportedToWavefrontRegistry() {
+		this.contextRunner
+				.withPropertyValues("wavefront.application.name=test-app", "wavefront.application.service=test-service",
+						"wavefront.application.cluster=test-cluster", "wavefront.application.shard=test-shard")
+				.with(wavefrontMetrics(() -> mock(WavefrontSender.class))).run((context) -> {
+					MeterRegistry registry = context.getBean(MeterRegistry.class);
+					registry.counter("my.counter", "env", "qa");
+					assertThat(registry.find("my.counter").tags("env", "qa").tags("application", "test-app")
+							.tags("service", "test-service").tags("cluster", "test-cluster").tags("shard", "test-shard")
+							.counter()).isNotNull();
+				});
+	}
+
+	@Test
+	void applicationTagsAreNotExportedToNonWavefrontRegistry() {
+		this.contextRunner
+				.withPropertyValues("wavefront.application.name=test-app", "wavefront.application.service=test-service")
+				.with(metrics()).withConfiguration(AutoConfigurations.of(SimpleMetricsExportAutoConfiguration.class))
+				.run((context) -> {
+					MeterRegistry registry = context.getBean(MeterRegistry.class);
+					registry.counter("my.counter", "env", "qa");
+					assertThat(registry.find("my.counter").tags("env", "qa")).isNotNull();
+					assertThat(registry.find("my.counter").tags("env", "qa").tags("application", "test-app")
+							.tags("service", "test-service").tags("cluster", "test-cluster").tags("shard", "test-shard")
+							.counter()).isNull();
+				});
+	}
+
+	@Test
+	void jvmReporterIsConfiguredWhenNoneExists() {
+		this.contextRunner.with(wavefrontMetrics(() -> mock(WavefrontSender.class)))
+				.run((context) -> assertThat(context).hasSingleBean(WavefrontJvmReporter.class));
+	}
+
+	@Test
+	void jvmReporterCanBeDisabled() {
+		this.contextRunner.withPropertyValues("wavefront.metrics.extract-jvm-metrics=false")
+				.with(wavefrontMetrics(() -> mock(WavefrontSender.class)))
+				.run(context -> assertThat(context).doesNotHaveBean(WavefrontJvmReporter.class));
+	}
+
+	@Test
+	void jvmReporterCanBeCustomized() {
+		WavefrontJvmReporter reporter = mock(WavefrontJvmReporter.class);
+		this.contextRunner.with(wavefrontMetrics(() -> mock(WavefrontSender.class)))
+				.withBean(WavefrontJvmReporter.class, () -> reporter)
+				.run((context) -> assertThat(context).getBean(WavefrontJvmReporter.class).isEqualTo(reporter));
+	}
+
+	@Test
+	void jvmReporterNotConfiguredWithoutWavefrontSender() {
+		this.contextRunner.with(metrics())
+				.run(context -> assertThat(context).doesNotHaveBean(WavefrontJvmReporter.class));
+	}
+
+	@Test
+	void tracingWithSleuthIsConfiguredWithWavefrontSender() {
+		WavefrontSender sender = mock(WavefrontSender.class);
+		this.contextRunner.withPropertyValues().with(wavefrontMetrics(() -> sender)).with(sleuth()).run((context) -> {
+			assertThat(context).hasSingleBean(TracingCustomizer.class);
+			WavefrontSleuthBraveSpanHandler braveSpanHandler = extractSpanHandler(context.getBean(Tracer.class));
+			assertThat(braveSpanHandler.spanHandler).hasFieldOrPropertyWithValue("wavefrontSender", sender);
+		});
+	}
+
+	@Test
+	void tracingWithSleuthWithEmptyEnvironmentUseDefaultTags() {
+		this.contextRunner.with(wavefrontMetrics(() -> mock(WavefrontSender.class))).with(sleuth())
+				.run(assertSleuthSpanDefaultTags("unnamed_application", "unnamed_service"));
+	}
+
+	@Test
+	void tracingWithSleuthWithWavefrontTagsAndSpringApplicationNameUseWavefrontTags() {
+		this.contextRunner
+				.withPropertyValues("wavefront.application.name=wavefront-application",
+						"wavefront.application.service=wavefront-service", "spring.application.name=spring-service")
+				.with(wavefrontMetrics(() -> mock(WavefrontSender.class))).with(sleuth())
+				.run(assertSleuthSpanDefaultTags("wavefront-application", "wavefront-service"));
+	}
+
+	@Test
+	void tracingWithSleuthWithSpringApplicationNameUseItRatherThanDefault() {
+		this.contextRunner.withPropertyValues("spring.application.name=spring-service")
+				.with(wavefrontMetrics(() -> mock(WavefrontSender.class))).with(sleuth())
+				.run(assertSleuthSpanDefaultTags("unnamed_application", "spring-service"));
+	}
+
+	@Test
+	void tracingWithSleuthWithCustomApplicationTagsUseThat() {
+		this.contextRunner
+				.withPropertyValues("wavefront.application.name=wavefront-application",
+						"wavefront.application.service=wavefront-service")
+				.with(wavefrontMetrics(() -> mock(WavefrontSender.class)))
+				.withBean(ApplicationTags.class,
+						() -> new ApplicationTags.Builder("custom-application", "custom-service")
+								.cluster("custom-cluster").shard("custom-shard").build())
+				.with(sleuth()).run(assertSleuthSpanDefaultTags("custom-application", "custom-service",
+						"custom-cluster", "custom-shard"));
+	}
+
+	@Test
+	void tracingWithSleuthWithCustomApplicationTagsAndEmptyValuesFallbackToDefaults() {
+		this.contextRunner
+				.withPropertyValues("wavefront.application.name=wavefront-application",
+						"wavefront.application.service=wavefront-service")
+				.with(wavefrontMetrics(() -> mock(WavefrontSender.class)))
+				.withBean(ApplicationTags.class,
+						() -> new ApplicationTags.Builder("custom-application", "custom-service").build())
+				.with(sleuth())
+				.run(assertSleuthSpanDefaultTags("custom-application", "custom-service", "none", "none"));
+	}
+
+	private ContextConsumer<AssertableApplicationContext> assertSleuthSpanDefaultTags(String applicationName,
+			String serviceName) {
+		return assertSleuthSpanDefaultTags(applicationName, serviceName, "none", "none");
+	}
+
+	private ContextConsumer<AssertableApplicationContext> assertSleuthSpanDefaultTags(String applicationName,
+			String serviceName, String cluster, String shard) {
+		return (context) -> {
+			assertThat(context).hasSingleBean(TracingCustomizer.class);
+			WavefrontSleuthBraveSpanHandler braveSpanHandler = extractSpanHandler(context.getBean(Tracer.class));
+			assertThat(braveSpanHandler.spanHandler.getDefaultTags()).contains(
+					new Pair<>("application", applicationName), new Pair<>("service", serviceName),
+					new Pair<>("cluster", cluster), new Pair<>("shard", shard));
+		};
+	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	void tracingWithSleuthCanBeConfigured() {
+		WavefrontSender sender = mock(WavefrontSender.class);
+		this.contextRunner.withPropertyValues()
+				.withPropertyValues("wavefront.tracing.red-metrics-custom-tag-keys=region,test")
+				.with(wavefrontMetrics(() -> sender)).with(sleuth()).run((context) -> {
+					assertThat(context).hasSingleBean(TracingCustomizer.class);
+					WavefrontSleuthBraveSpanHandler braveSpanHandler = extractSpanHandler(
+							context.getBean(Tracer.class));
+					WavefrontSleuthSpanHandler spanHandler = braveSpanHandler.spanHandler;
+					Set<String> traceDerivedCustomTagKeys = (Set<String>) ReflectionTestUtils.getField(spanHandler,
+							"traceDerivedCustomTagKeys");
+					assertThat(traceDerivedCustomTagKeys).containsExactlyInAnyOrder("region", "test");
+				});
+	}
+
+	@Test
+	void tracingWithOpenTracingBacksOffWhenSpringCloudSleuthIsAvailable() {
+		this.contextRunner.with(wavefrontMetrics(() -> mock(WavefrontSender.class)))
+				.run((context) -> assertThat(context).hasSingleBean(TracingCustomizer.class)
+						.doesNotHaveBean(io.opentracing.Tracer.class));
+	}
+
+	@Test
+	void tracingIsDisabledWhenOpenTracingAndSleuthAreNotAvailable() {
+		new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(WavefrontAutoConfiguration.class))
+				.withClassLoader(new FilteredClassLoader("org.springframework.cloud.sleuth", "io.opentracing"))
+				.with(wavefrontMetrics(() -> mock(WavefrontSender.class))).run((context) -> assertThat(context)
+						.doesNotHaveBean(TracingCustomizer.class).doesNotHaveBean(io.opentracing.Tracer.class));
+	}
+
+	@Test
+	void tracingCanBeDisabled() {
+		this.contextRunner.withPropertyValues("wavefront.tracing.enabled=false")
+				.with(wavefrontMetrics(() -> mock(WavefrontSender.class))).run((context) -> assertThat(context)
+						.doesNotHaveBean(TracingCustomizer.class).doesNotHaveBean(io.opentracing.Tracer.class));
+	}
+
+	@Test
+	void tracingIsNotConfiguredWithNonWavefrontRegistry() {
+		this.contextRunner.with(metrics()).run((context) -> assertThat(context).doesNotHaveBean(Tracer.class));
+	}
+
+	@SuppressWarnings("ConstantConditions")
+	private WavefrontSleuthBraveSpanHandler extractSpanHandler(Tracer tracer) {
+		SpanHandler[] handlers = (SpanHandler[]) ReflectionTestUtils.getField(
+				ReflectionTestUtils.getField(ReflectionTestUtils.getField(tracer, "spanHandler"), "delegate"),
+				"handlers");
+		return (WavefrontSleuthBraveSpanHandler) handlers[1];
+	}
+
+	@SuppressWarnings("unchecked")
+	private static <T extends AbstractApplicationContextRunner<?, ?, ?>> Function<T, T> wavefrontMetrics(
+			Supplier<WavefrontSender> wavefrontSender) {
+		return (runner) -> (T) runner.withBean(WavefrontSender.class, wavefrontSender)
+				.withConfiguration(AutoConfigurations.of(WavefrontMetricsExportAutoConfiguration.class))
+				.with(metrics());
+	}
+
+	@SuppressWarnings("unchecked")
+	private static <T extends AbstractApplicationContextRunner<?, ?, ?>> Function<T, T> metrics() {
+		return (runner) -> (T) runner.withPropertyValues("management.metrics.use-global-registry=false")
+				.withConfiguration(AutoConfigurations.of(MetricsAutoConfiguration.class,
+						CompositeMeterRegistryAutoConfiguration.class));
+	}
+
+	@SuppressWarnings("unchecked")
+	private static <T extends AbstractApplicationContextRunner<?, ?, ?>> Function<T, T> sleuth() {
+		return (runner) -> (T) runner.withConfiguration(AutoConfigurations.of(BraveAutoConfiguration.class));
+	}
+
+}

spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontTracingIntegrationTests.java

@@ -0,0 +1,369 @@
+/*
+ * Copyright 2013-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.sleuth.autoconfig.wavefront;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.TimeUnit;
+
+import brave.Tracing;
+import brave.internal.Platform;
+import brave.opentracing.BraveTracer;
+import brave.sampler.Sampler;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.wavefront.sdk.entities.histograms.HistogramGranularity;
+import com.wavefront.sdk.entities.tracing.SpanLog;
+import io.opentracing.Tracer;
+import io.opentracing.tag.Tags;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.test.autoconfigure.actuate.metrics.AutoConfigureMetrics;
+import org.springframework.boot.test.autoconfigure.web.reactive.AutoConfigureWebTestClient;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Controller;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.web.reactive.server.WebTestClient;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Integration tests for tracing.
+ */
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT,
+		classes = WavefrontTracingIntegrationTests.Config.class,
+		properties = { "wavefront.application.name=IntegratedTracingTests", "spring.application.name=test_service" })
+@AutoConfigureWebTestClient
+@AutoConfigureMetrics
+@DirtiesContext
+public class WavefrontTracingIntegrationTests {
+
+	@Autowired
+	private WebTestClient client;
+
+	@Autowired
+	private BlockingDeque<SpanRecord> spanRecordQueue;
+
+	@Test
+	void sendsToWavefront() {
+		this.client.get().uri("/api/fn/10").header("b3", "0000000000000001-0000000000000003-1-0000000000000002")
+				.exchange().expectStatus().isOk();
+
+		SpanRecord spanRecord = takeRecord(spanRecordQueue);
+		assertThat(spanRecord.traceId).hasToString("00000000-0000-0000-0000-000000000001");
+		assertThat(spanRecord.parents).extracting(UUID::toString)
+				.containsExactly("00000000-0000-0000-0000-000000000003");
+		assertThat(spanRecord.followsFrom).isNull();
+		// This tests that RPC spans do not share the same span ID
+		assertThat(spanRecord.spanId.toString()).isNotEqualTo("00000000-0000-0000-0000-000000000003")
+				.matches("^[0-9a-f]{8}\\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\\b[0-9a-f]{12}$");
+		assertThat(spanRecord.name).isEqualTo("GET /api/fn/{id}");
+
+		// spot check the unit is valid (millis not micros)
+		long currentTime = System.currentTimeMillis();
+		assertThat(spanRecord.startMillis).isGreaterThan(currentTime - 5000).isLessThan(currentTime);
+		// Less than a millis should round up to 1, but the test could take longer than
+		// 1ms
+		assertThat(spanRecord.durationMillis).isPositive();
+
+		// http
+		assertThat(spanRecord.tags).containsExactlyInAnyOrder(Pair.of("application", "IntegratedTracingTests"),
+				Pair.of("service", "test_service"), Pair.of("cluster", "none"), Pair.of("shard", "none"),
+				Pair.of("http.method", "GET"), Pair.of("http.path", "/api/fn/10"),
+				Pair.of("mvc.controller.class", "WebMvcController"), Pair.of("mvc.controller.method", "fn"),
+				Pair.of("span.kind", "server"), Pair.of("ipv4", Platform.get().linkLocalIp()));
+	}
+
+	@Test
+	void http_badRequest_setsStatusCodeAndErrorTrueTags() {
+		this.client.get().uri("/badrequest").exchange().expectStatus().isBadRequest();
+
+		SpanRecord spanRecord = takeRecord(spanRecordQueue);
+
+		// http
+		assertThat(spanRecord.tags).contains(Pair.of("http.status_code", "400"), Pair.of("error", "true"));
+	}
+
+	@Test
+	void setsStatusCodeAndErrorTrueTags_opentracing() {
+		this.client.get().uri("/error/opentracing").exchange().expectStatus().is5xxServerError();
+
+		SpanRecord spanRecord = takeRecord(spanRecordQueue);
+		// http
+		assertThat(spanRecord.tags).contains(Pair.of("http.status_code", "500"), Pair.of("error", "true") // retains
+																											// the
+																											// boolean
+																											// true
+		);
+	}
+
+	@Test
+	void setsStatusCodeAndErrorTrueTags_brave() {
+		this.client.get().uri("/error/brave").exchange().expectStatus().is5xxServerError();
+
+		SpanRecord spanRecord = takeRecord(spanRecordQueue);
+		// http
+		assertThat(spanRecord.tags).contains(Pair.of("http.status_code", "500"), Pair.of("error", "true") // deletes
+																											// the
+																											// user
+																											// message
+		);
+	}
+
+	@Test
+	void setsStatusCodeAndErrorTrueTags_exception() {
+		this.client.get().uri("/error/exception").exchange().expectStatus().is5xxServerError();
+
+		// http
+		assertThat(takeRecord(spanRecordQueue).tags).contains(Pair.of("http.status_code", "500"),
+				Pair.of("error", "true") // deletes the exception message
+		);
+	}
+
+	@Test
+	void setsStatusCodeAndErrorTrueTags_thrownException() {
+		this.client.get().uri("/throws").exchange().expectStatus().is5xxServerError();
+
+		SpanRecord spanRecord = takeRecord(spanRecordQueue);
+		// http
+		assertThat(spanRecord.tags).contains(
+				/* Pair.of("http.status_code", "500"), */ // Able to return error=true
+															// span tag but not
+															// http.status_code span tag
+				Pair.of("error", "true") // deletes the exception message
+		);
+	}
+
+	/** Helps ensure test bugs don't result in hung tests! */
+	<R> R takeRecord(BlockingDeque<R> queue) {
+		R result;
+		try {
+			result = queue.poll(3, TimeUnit.SECONDS);
+		}
+		catch (InterruptedException e) {
+			Thread.currentThread().interrupt();
+			throw new AssertionError(e);
+		}
+
+		assertThat(result).withFailMessage("Record was not reported").isNotNull();
+		return result;
+	}
+
+	/** Makes sure tests aren't accidentally not verifying all reported data. */
+	@AfterEach
+	void ensureNoExtraSpans() {
+		try {
+			SpanRecord span = spanRecordQueue.poll(100, TimeUnit.MILLISECONDS);
+			assertThat(span).withFailMessage("Span remaining in queue. Check for redundant reporting: %s", span)
+					.isNull();
+		}
+		catch (InterruptedException e) {
+			Thread.currentThread().interrupt();
+			throw new AssertionError(e);
+		}
+	}
+
+	@Configuration
+	@EnableAutoConfiguration
+	static class Config {
+
+		/**
+		 * This uses a {@linkplain Controller WebMVC controller} as it is the most popular
+		 * way to write Spring services and has no instrumentation gotchas or scope bugs
+		 * like reactive tracing. This allows us to focus on api and data mapping issues,
+		 * which is the heart of this test.
+		 */
+		@Bean
+		WebMvcController controller() {
+			return new WebMvcController();
+		}
+
+		@Bean
+		Sampler sampler() {
+			return Sampler.ALWAYS_SAMPLE;
+		}
+
+		/**
+		 * Sleuth would automatically wire this, except there's another impl in the
+		 * classpath.
+		 */
+		@Bean
+		Tracer opentracing(Tracing tracing) {
+			return BraveTracer.create(tracing);
+		}
+
+		@Bean
+		BlockingDeque<SpanRecord> spanRecordQueue() {
+			return new LinkedBlockingDeque<>();
+		}
+
+		@Bean
+		@Primary
+		WavefrontSender wavefrontSender(BlockingDeque<SpanRecord> spanRecordQueue) {
+			return new WavefrontSender() {
+				@Override
+				public String getClientId() {
+					return null;
+				}
+
+				@Override
+				public void flush() {
+
+				}
+
+				@Override
+				public int getFailureCount() {
+					return 0;
+				}
+
+				@Override
+				public void sendDistribution(String name, List<Pair<Double, Integer>> centroids,
+						Set<HistogramGranularity> histogramGranularities, Long timestamp, String source,
+						Map<String, String> tags) {
+
+				}
+
+				@Override
+				public void sendMetric(String name, double value, Long timestamp, String source,
+						Map<String, String> tags) {
+
+				}
+
+				@Override
+				public void sendFormattedMetric(String point) {
+
+				}
+
+				@Override
+				public void sendSpan(String name, long startMillis, long durationMillis, String source, UUID traceId,
+						UUID spanId, List<UUID> parents, List<UUID> followsFrom, List<Pair<String, String>> tags,
+						List<SpanLog> spanLogs) {
+					spanRecordQueue.add(new SpanRecord(name, startMillis, durationMillis, source, traceId, spanId,
+							parents, followsFrom, tags, spanLogs));
+				}
+
+				@Override
+				public void close() {
+
+				}
+			};
+		}
+
+	}
+
+	@Controller
+	static class WebMvcController {
+
+		@Autowired
+		Tracer opentracing;
+
+		@Autowired
+		brave.Tracer tracer;
+
+		@RequestMapping("/api/fn/{id}")
+		public ResponseEntity<String> fn(@PathVariable("id") String id) {
+			return new ResponseEntity<>(id, HttpStatus.OK);
+		}
+
+		@RequestMapping("/error/{api}")
+		public ResponseEntity<Void> error(@PathVariable("api") String api) {
+			switch (api) {
+			case "brave":
+				tracer.currentSpanCustomizer().tag("error", "user message");
+				break;
+			case "opentracing":
+				opentracing.activeSpan().setTag(Tags.ERROR, true);
+				break;
+			case "exception":
+				tracer.currentSpan().error(new RuntimeException("uncaught!"));
+				break;
+			default:
+				return new ResponseEntity<>(HttpStatus.BAD_REQUEST);
+			}
+			return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
+		}
+
+		@RequestMapping("/badrequest")
+		public ResponseEntity<Void> badrequest() {
+			return new ResponseEntity<>(HttpStatus.BAD_REQUEST);
+		}
+
+		@RequestMapping("throws")
+		public void toss() {
+			throw new IllegalStateException("boom");
+		}
+
+	}
+
+	static final class SpanRecord {
+
+		private final String name;
+
+		private final long startMillis;
+
+		private final long durationMillis;
+
+		private final String source;
+
+		private final UUID traceId;
+
+		private final UUID spanId;
+
+		private final List<UUID> parents;
+
+		private final List<UUID> followsFrom;
+
+		private final List<Pair<String, String>> tags;
+
+		private final List<SpanLog> spanLogs;
+
+		SpanRecord(String name, long startMillis, long durationMillis, String source, UUID traceId, UUID spanId,
+				List<UUID> parents, List<UUID> followsFrom, List<Pair<String, String>> tags, List<SpanLog> spanLogs) {
+			this.name = name;
+			this.startMillis = startMillis;
+			this.durationMillis = durationMillis;
+			this.source = source;
+			this.traceId = traceId;
+			this.spanId = spanId;
+			this.parents = parents;
+			this.followsFrom = followsFrom;
+			this.tags = tags;
+			this.spanLogs = spanLogs;
+		}
+
+		@Override
+		public String toString() {
+			return "SpanRecord{" + "name='" + name + '\'' + ", traceId=" + traceId + ", spanId=" + spanId + '}';
+		}
+
+	}
+
+}

spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/zipkin2/ZipkinSamplerTests.java

@@ -29,6 +29,8 @@ import org.springframework.boot.test.context.runner.ApplicationContextRunner;
 import org.springframework.cloud.gateway.config.GatewayAutoConfiguration;
 import org.springframework.cloud.gateway.config.GatewayClassPathWarningAutoConfiguration;
 import org.springframework.cloud.gateway.config.GatewayMetricsAutoConfiguration;
+import org.springframework.cloud.sleuth.autoconfig.NoOpWavefrontSender;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
 /**
@@ -56,6 +58,11 @@ public class ZipkinSamplerTests {
 			MongoAutoConfiguration.class, QuartzAutoConfiguration.class })
 	static class TestConfig {
 
+		@Bean
+		NoOpWavefrontSender noOpWavefrontSender() {
+			return new NoOpWavefrontSender();
+		}
+
 	}
 
 }

spring-cloud-sleuth-autoconfigure/src/test/resources/application.yml

@@ -1,3 +1,7 @@
 logging.level.org.springframework.cloud: DEBUG
 
 spring.autoconfigure.exclude: org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration, org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration, org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration, org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration, org.springframework.cloud.gateway.config.GatewayAutoConfiguration, org.springframework.cloud.gateway.config.GatewayClassPathWarningAutoConfiguration, org.springframework.cloud.gateway.config.GatewayMetricsAutoConfiguration, org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration
+
+wavefront:
+  metrics:
+    extract-jvm-metrics: false