From a1c136b69fddff28839f867df8299034a5ab4d6f Mon Sep 17 00:00:00 2001 From: Marcin Grzejszczak Date: Thu, 11 Feb 2021 10:43:05 +0100 Subject: [PATCH] WIP --- pom.xml | 8 + spring-cloud-sleuth-autoconfigure/pom.xml | 28 ++ .../wavefront/WavefrontProperties.java | 59 +++ .../WavefrontSleuthAutoConfiguration.java | 90 ++++ .../WavefrontSleuthBraveSpanHandler.java | 52 ++ .../wavefront/WavefrontSleuthSpanHandler.java | 467 ++++++++++++++++++ .../main/resources/META-INF/spring.factories | 3 +- .../autoconfig/NoOpWavefrontSender.java | 78 +++ .../TraceNoOpAutoConfiguration.java | 10 +- .../WavefrontAutoConfigurationTests.java | 324 ++++++++++++ .../WavefrontTracingIntegrationTests.java | 369 ++++++++++++++ .../zipkin2/ZipkinSamplerTests.java | 7 + .../src/test/resources/application.yml | 4 + 13 files changed, 1497 insertions(+), 2 deletions(-) create mode 100644 spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontProperties.java create mode 100644 spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthAutoConfiguration.java create mode 100644 spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthBraveSpanHandler.java create mode 100644 spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthSpanHandler.java create mode 100644 spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/NoOpWavefrontSender.java create mode 100644 spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontAutoConfigurationTests.java create mode 100644 spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontTracingIntegrationTests.java diff --git a/pom.xml b/pom.xml index f929e4aa5..129c32c7a 100644 --- a/pom.xml +++ b/pom.xml @@ -71,6 +71,7 @@ 3.0.2-SNAPSHOT 3.0.2-SNAPSHOT 5.13.2 + 2.1.1-SNAPSHOT 0.32.0 2.3.4.RELEASE false @@ -228,6 +229,13 @@ pom import + + com.wavefront + wavefront-spring-boot-bom + ${wavefront-spring-boot.version} + pom + import + org.springframework.security.oauth.boot spring-security-oauth2-autoconfigure diff --git a/spring-cloud-sleuth-autoconfigure/pom.xml b/spring-cloud-sleuth-autoconfigure/pom.xml index 6f3aa9fc0..df682a6d9 100644 --- a/spring-cloud-sleuth-autoconfigure/pom.xml +++ b/spring-cloud-sleuth-autoconfigure/pom.xml @@ -365,6 +365,34 @@ + + + com.wavefront + wavefront-opentracing-sdk-java + true + + + com.wavefront + wavefront-spring-boot + true + + + com.wavefront + wavefront-runtime-sdk-jvm + true + + + com.wavefront + wavefront-sdk-java + true + + + io.micrometer + micrometer-registry-wavefront + true + + + org.springframework.boot spring-boot-starter-test diff --git a/spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontProperties.java b/spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontProperties.java new file mode 100644 index 000000000..7351e8cd4 --- /dev/null +++ b/spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontProperties.java @@ -0,0 +1,59 @@ +/* + * Copyright 2013-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.sleuth.autoconfig.wavefront; + +import java.util.HashSet; +import java.util.Set; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Advanced configuration properties for Wavefront. + * + * @author Stephane Nicoll + */ +@ConfigurationProperties("wavefront.tracing") +public class WavefrontProperties { + + /** + * Enables Wavefront tracing. + */ + private boolean enabled; + + /** + * Tags that should be associated with RED metrics. If the span has any of the + * specified tags, then those get reported to generated RED metrics. + */ + private Set redMetricsCustomTagKeys = new HashSet<>(); + + public Set getRedMetricsCustomTagKeys() { + return this.redMetricsCustomTagKeys; + } + + public void setRedMetricsCustomTagKeys(Set redMetricsCustomTagKeys) { + this.redMetricsCustomTagKeys = redMetricsCustomTagKeys; + } + + public boolean isEnabled() { + return this.enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + +} diff --git a/spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthAutoConfiguration.java b/spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthAutoConfiguration.java new file mode 100644 index 000000000..137dd8659 --- /dev/null +++ b/spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthAutoConfiguration.java @@ -0,0 +1,90 @@ +/* + * Copyright 2013-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.sleuth.autoconfig.wavefront; + +import brave.Tracer; +import brave.TracingCustomizer; +import brave.handler.SpanHandler; +import com.wavefront.sdk.common.WavefrontSender; +import com.wavefront.sdk.common.application.ApplicationTags; +import com.wavefront.spring.autoconfigure.WavefrontAutoConfiguration; +import com.wavefront.spring.autoconfigure.WavefrontTracerBla; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.wavefront.WavefrontConfig; + +import org.springframework.boot.actuate.autoconfigure.metrics.export.wavefront.WavefrontMetricsExportAutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.sleuth.SpanNamer; +import org.springframework.cloud.sleuth.autoconfig.brave.BraveAutoConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Configuration for Wavefront tracing using Spring Cloud Sleuth. + * + * @author Adrian Cole + * @author Stephane Nicoll + * @since 3.1.0 + */ +@Configuration(proxyBeanMethods = false) +@ConditionalOnClass({ SpanNamer.class, MeterRegistry.class, WavefrontConfig.class, WavefrontSender.class }) +@ConditionalOnBean(WavefrontSender.class) +@AutoConfigureBefore({ BraveAutoConfiguration.class, WavefrontAutoConfiguration.class }) +@AutoConfigureAfter(WavefrontMetricsExportAutoConfiguration.class) +@EnableConfigurationProperties(WavefrontProperties.class) +@ConditionalOnProperty(value = "wavefront.tracing.enabled", matchIfMissing = true) +public class WavefrontSleuthAutoConfiguration { + + static final String BEAN_NAME = "wavefrontTracingCustomizer"; + + @Bean + @ConditionalOnBean({ MeterRegistry.class, WavefrontConfig.class, WavefrontSender.class }) + WavefrontSleuthSpanHandler wavefrontSleuthSpanHandler(MeterRegistry meterRegistry, WavefrontSender wavefrontSender, + ApplicationTags applicationTags, WavefrontConfig wavefrontConfig, WavefrontProperties wavefrontProperties) { + return new WavefrontSleuthSpanHandler( + // https://github.com/wavefrontHQ/wavefront-opentracing-sdk-java/blob/f1f08d8daf7b692b9b61dcd5bc24ca6befa8e710/src/main/java/com/wavefront/opentracing/reporting/WavefrontSpanReporter.java#L54 + 50000, // TODO: maxQueueSize should be a property, ya? + wavefrontSender, meterRegistry, wavefrontConfig.source(), applicationTags, wavefrontProperties); + } + + @Configuration(proxyBeanMethods = false) + @ConditionalOnClass({ Tracer.class, TracingCustomizer.class, SpanHandler.class }) + static class BraveCustomizerConfiguration { + + @Bean(BEAN_NAME) + @ConditionalOnMissingBean(name = BEAN_NAME) + @ConditionalOnBean({ MeterRegistry.class, WavefrontConfig.class, WavefrontSender.class }) + TracingCustomizer wavefrontTracingCustomizer(WavefrontSleuthSpanHandler spanHandler) { + return t -> t.traceId128Bit(true).supportsJoin(false) + .addSpanHandler(new WavefrontSleuthBraveSpanHandler(spanHandler)); + } + + @Bean + WavefrontTracerBla wavefrontTracerBla() { + return new WavefrontTracerBla() { + }; + } + + } + +} diff --git a/spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthBraveSpanHandler.java b/spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthBraveSpanHandler.java new file mode 100644 index 000000000..b7024f20f --- /dev/null +++ b/spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthBraveSpanHandler.java @@ -0,0 +1,52 @@ +/* + * Copyright 2013-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.sleuth.autoconfig.wavefront; + +import java.io.Closeable; +import java.io.IOException; + +import brave.handler.MutableSpan; +import brave.handler.SpanHandler; +import brave.propagation.TraceContext; + +import org.springframework.cloud.sleuth.brave.bridge.BraveFinishedSpan; +import org.springframework.cloud.sleuth.brave.bridge.BraveTraceContext; + +class WavefrontSleuthBraveSpanHandler extends SpanHandler implements Runnable, Closeable { + + final WavefrontSleuthSpanHandler spanHandler; + + WavefrontSleuthBraveSpanHandler(WavefrontSleuthSpanHandler spanHandler) { + this.spanHandler = spanHandler; + } + + @Override + public boolean end(TraceContext context, MutableSpan span, Cause cause) { + return spanHandler.end(BraveTraceContext.fromBrave(context), BraveFinishedSpan.fromBrave(span)); + } + + @Override + public void close() throws IOException { + this.spanHandler.close(); + } + + @Override + public void run() { + this.spanHandler.run(); + } + +} diff --git a/spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthSpanHandler.java b/spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthSpanHandler.java new file mode 100644 index 000000000..da70a69aa --- /dev/null +++ b/spring-cloud-sleuth-autoconfigure/src/main/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontSleuthSpanHandler.java @@ -0,0 +1,467 @@ +/* + * Copyright 2013-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.sleuth.autoconfig.wavefront; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import com.wavefront.internal.reporter.WavefrontInternalReporter; +import com.wavefront.java_sdk.com.google.common.collect.Iterators; +import com.wavefront.java_sdk.com.google.common.collect.Sets; +import com.wavefront.sdk.common.NamedThreadFactory; +import com.wavefront.sdk.common.Pair; +import com.wavefront.sdk.common.WavefrontSender; +import com.wavefront.sdk.common.application.ApplicationTags; +import com.wavefront.sdk.entities.tracing.SpanLog; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.cloud.sleuth.TraceContext; +import org.springframework.cloud.sleuth.exporter.FinishedSpan; +import org.springframework.util.StringUtils; + +import static com.wavefront.internal.SpanDerivedMetricsUtils.TRACING_DERIVED_PREFIX; +import static com.wavefront.internal.SpanDerivedMetricsUtils.reportHeartbeats; +import static com.wavefront.internal.SpanDerivedMetricsUtils.reportWavefrontGeneratedData; +import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY; +import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY; +import static com.wavefront.sdk.common.Constants.COMPONENT_TAG_KEY; +import static com.wavefront.sdk.common.Constants.DEBUG_TAG_KEY; +import static com.wavefront.sdk.common.Constants.ERROR_TAG_KEY; +import static com.wavefront.sdk.common.Constants.NULL_TAG_VAL; +import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY; +import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY; +import static com.wavefront.sdk.common.Constants.SOURCE_KEY; +import static com.wavefront.sdk.common.Constants.SPAN_LOG_KEY; + +/** + * This converts a span recorded by Brave and invokes {@link WavefrontSender#sendSpan}. + * + *

+ * This uses a combination of conversion approaches from Wavefront projects: + *

    + *
  • https://github.com/wavefrontHQ/wavefront-opentracing-sdk-java
    • + *
  • https://github.com/wavefrontHQ/wavefront-proxy
    • + *
+ * + *

+ * On conflict, we make a comment and prefer wavefront-opentracing-sdk-java. The rationale + * is wavefront-opentracing-sdk-java uses the same {@link WavefrontSender#sendSpan} + * library, so it is easier to reason with. This policy can be revisited by future + * maintainers. + * + *

+ * Note:UUID conversions follow the same conventions used in practice in + * Wavefront. Ex. + * https://github.com/wavefrontHQ/wavefront-opentracing-sdk-java/blob/6babf2ff95daa37452e1e8c35ae54b58b6abb50f/src/main/java/com/wavefront/opentracing/propagation/JaegerWavefrontPropagator.java#L191-L204 + * While in practice this is not a problem, it is worth mentioning that this convention + * will only only result in RFC 4122 timestamp (version 1) format by accident. In other + * words, don't call {@link UUID#timestamp()} on UUIDs converted here, or in other + * Wavefront code, as it might throw. + * + * @since 3.1.0 + */ +public final class WavefrontSleuthSpanHandler implements Runnable, Closeable { + + private static final Log LOG = LogFactory.getLog(WavefrontSleuthSpanHandler.class); + + // https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L114-L114 + private static final String DEFAULT_SPAN_NAME = "defaultOperation"; + + private final static String DEFAULT_SOURCE = "wavefront-spring-boot"; + + private final static String WAVEFRONT_GENERATED_COMPONENT = "wavefront-generated"; + + private static final int LONG_BYTES = Long.SIZE / Byte.SIZE; + + private static final int BYTE_BASE16 = 2; + + private static final int LONG_BASE16 = BYTE_BASE16 * LONG_BYTES; + + private static final int TRACE_ID_HEX_SIZE = 2 * LONG_BASE16; + + private static final String ALPHABET = "0123456789abcdef"; + + private static final int ASCII_CHARACTERS = 128; + + private static final byte[] DECODING = buildDecodingArray(); + + final LinkedBlockingQueue> spanBuffer; + + final WavefrontSender wavefrontSender; + + final WavefrontInternalReporter wfInternalReporter; + + final Set traceDerivedCustomTagKeys; + + final Counter spansDropped; + + final Counter spansReceived; + + final Counter reportErrors; + + final Thread sendingThread; + + private volatile boolean stop = false; + + private final Set, String>> discoveredHeartbeatMetrics; + + private final ScheduledExecutorService heartbeatMetricsScheduledExecutorService; + + final String source; + + final List> defaultTags; + + final Set defaultTagKeys; + + final ApplicationTags applicationTags; + + WavefrontSleuthSpanHandler(int maxQueueSize, WavefrontSender wavefrontSender, MeterRegistry meterRegistry, + String source, ApplicationTags applicationTags, WavefrontProperties wavefrontProperties) { + this.wavefrontSender = wavefrontSender; + this.applicationTags = applicationTags; + this.discoveredHeartbeatMetrics = Sets.newConcurrentHashSet(); + + this.heartbeatMetricsScheduledExecutorService = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("sleuth-heart-beater").setDaemon(true)); + + // Emit Heartbeats Metrics every 1 min. + heartbeatMetricsScheduledExecutorService.scheduleAtFixedRate(() -> { + try { + reportHeartbeats(wavefrontSender, discoveredHeartbeatMetrics, WAVEFRONT_GENERATED_COMPONENT); + } + catch (IOException e) { + LOG.warn("Cannot report heartbeat metric to wavefront"); + } + }, 1, 60, TimeUnit.SECONDS); + + this.traceDerivedCustomTagKeys = new HashSet<>(wavefrontProperties.getRedMetricsCustomTagKeys()); + + // Start the reporter + wfInternalReporter = new WavefrontInternalReporter.Builder().prefixedWith(TRACING_DERIVED_PREFIX) + .withSource(DEFAULT_SOURCE).reportMinuteDistribution().build(wavefrontSender); + wfInternalReporter.start(1, TimeUnit.MINUTES); + + this.source = source; + this.defaultTags = createDefaultTags(applicationTags); + this.defaultTagKeys = defaultTags.stream().map(p -> p._1).collect(Collectors.toSet()); + this.defaultTagKeys.add(SOURCE_KEY); + + this.spanBuffer = new LinkedBlockingQueue<>(maxQueueSize); + + // init internal metrics + meterRegistry.gauge("reporter.queue.size", spanBuffer, sb -> (double) sb.size()); + meterRegistry.gauge("reporter.queue.remaining_capacity", spanBuffer, sb -> (double) sb.remainingCapacity()); + this.spansReceived = meterRegistry.counter("reporter.spans.received"); + this.spansDropped = meterRegistry.counter("reporter.spans.dropped"); + this.reportErrors = meterRegistry.counter("reporter.errors"); + + this.sendingThread = new Thread(this, "wavefrontSpanReporter"); + this.sendingThread.setDaemon(true); + this.sendingThread.start(); + } + + // Exact same behavior as WavefrontSpanReporter + // https://github.com/wavefrontHQ/wavefront-opentracing-sdk-java/blob/f1f08d8daf7b692b9b61dcd5bc24ca6befa8e710/src/main/java/com/wavefront/opentracing/reporting/WavefrontSpanReporter.java#L163-L179 + public boolean end(TraceContext context, FinishedSpan span) { + spansReceived.increment(); + if (!spanBuffer.offer(Pair.of(context, span))) { + spansDropped.increment(); + if (LOG.isWarnEnabled()) { + LOG.warn("Buffer full, dropping span: " + span); + LOG.warn("Total spans dropped: " + spansDropped.count()); + } + } + return true; // regardless of error, other handlers should run + } + + List> getDefaultTags() { + return Collections.unmodifiableList(this.defaultTags); + } + + private String padLeftWithZeros(String string, int length) { + if (string.length() >= length) { + return string; + } + else { + StringBuilder sb = new StringBuilder(length); + for (int i = string.length(); i < length; i++) { + sb.append('0'); + } + + return sb.append(string).toString(); + } + } + + private void send(TraceContext context, FinishedSpan span) { + String traceIdString = padLeftWithZeros(context.traceId(), TRACE_ID_HEX_SIZE); + String traceIdHigh = traceIdString.substring(0, traceIdString.length() / 2); + String traceIdLow = traceIdString.substring(traceIdString.length() / 2); + UUID traceId = new UUID(longFromBase16String(traceIdHigh), longFromBase16String(traceIdLow)); + UUID spanId = new UUID(0L, longFromBase16String(context.spanId())); + + // NOTE: wavefront-opentracing-sdk-java and wavefront-proxy differ, but we prefer + // the former. + // https://github.com/wavefrontHQ/wavefront-opentracing-sdk-java/blob/f1f08d8daf7b692b9b61dcd5bc24ca6befa8e710/src/main/java/com/wavefront/opentracing/reporting/WavefrontSpanReporter.java#L187-L190 + // https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L248-L252 + List parents = null; + String parentId = context.parentId(); + if (StringUtils.hasText(parentId) && longFromBase16String(parentId) != 0L) { + parents = Collections.singletonList(new UUID(0L, longFromBase16String(parentId))); + } + List followsFrom = null; + + // https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L344-L345 + String name = span.getName(); + if (name == null) { + name = DEFAULT_SPAN_NAME; + } + + // Start and duration become 0L if unset. Any positive duration rounds up to 1 + // millis. + long startMillis = span.getStartTimestamp() / 1000L; + long finishMillis = span.getEndTimestamp() / 1000L; + long durationMicros = span.getEndTimestamp() - span.getStartTimestamp(); + long durationMillis = startMillis != 0 && finishMillis != 0L ? Math.max(finishMillis - startMillis, 1L) : 0L; + + List spanLogs = convertAnnotationsToSpanLogs(span); + TagList tags = new TagList(defaultTagKeys, defaultTags, span); + + try { + wavefrontSender.sendSpan(name, startMillis, durationMillis, source, traceId, spanId, parents, followsFrom, + tags, spanLogs); + } + catch (IOException | RuntimeException t) { + if (LOG.isDebugEnabled()) { + LOG.debug("error sending span " + context, t); + } + } + + // report stats irrespective of span sampling. + if (wfInternalReporter != null) { + // report converted metrics/histograms from the span + try { + discoveredHeartbeatMetrics.add(reportWavefrontGeneratedData(wfInternalReporter, name, + applicationTags.getApplication(), applicationTags.getService(), + applicationTags.getCluster() == null ? NULL_TAG_VAL : applicationTags.getCluster(), + applicationTags.getShard() == null ? NULL_TAG_VAL : applicationTags.getShard(), source, + tags.componentTagValue, tags.isError, durationMicros, traceDerivedCustomTagKeys, tags)); + } + catch (RuntimeException t) { + if (LOG.isDebugEnabled()) { + LOG.debug("error sending span RED metrics " + context, t); + } + } + } + } + + private static byte[] buildDecodingArray() { + byte[] decoding = new byte[ASCII_CHARACTERS]; + Arrays.fill(decoding, (byte) -1); + for (int i = 0; i < ALPHABET.length(); i++) { + char c = ALPHABET.charAt(i); + decoding[c] = (byte) i; + } + return decoding; + } + + /** + * Returns the {@code long} value whose base16 representation is stored in the first + * 16 chars of {@code chars} starting from the {@code offset}. + * @param chars the base16 representation of the {@code long}. + */ + private static long longFromBase16String(CharSequence chars) { + int offset = 0; + return (decodeByte(chars.charAt(offset), chars.charAt(offset + 1)) & 0xFFL) << 56 + | (decodeByte(chars.charAt(offset + 2), chars.charAt(offset + 3)) & 0xFFL) << 48 + | (decodeByte(chars.charAt(offset + 4), chars.charAt(offset + 5)) & 0xFFL) << 40 + | (decodeByte(chars.charAt(offset + 6), chars.charAt(offset + 7)) & 0xFFL) << 32 + | (decodeByte(chars.charAt(offset + 8), chars.charAt(offset + 9)) & 0xFFL) << 24 + | (decodeByte(chars.charAt(offset + 10), chars.charAt(offset + 11)) & 0xFFL) << 16 + | (decodeByte(chars.charAt(offset + 12), chars.charAt(offset + 13)) & 0xFFL) << 8 + | (decodeByte(chars.charAt(offset + 14), chars.charAt(offset + 15)) & 0xFFL); + } + + private static byte decodeByte(char hi, char lo) { + int decoded = DECODING[hi] << 4 | DECODING[lo]; + return (byte) decoded; + } + + @Override + public void run() { + while (!stop) { + try { + Pair contextAndSpan = spanBuffer.take(); + send(contextAndSpan._1, contextAndSpan._2); + } + catch (InterruptedException ex) { + if (LOG.isInfoEnabled()) { + LOG.info("reporting thread interrupted"); + } + } + catch (Throwable ex) { + LOG.warn("Error processing buffer", ex); + } + } + } + + @Override + public void close() { + stop = true; + try { + // wait for 5 secs max + sendingThread.join(5000); + heartbeatMetricsScheduledExecutorService.shutdownNow(); + } + catch (InterruptedException ex) { + // no-op + } + } + + // https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L397-L402 + static List convertAnnotationsToSpanLogs(FinishedSpan span) { + int annotationCount = span.getEvents().size(); + if (annotationCount == 0) { + return Collections.emptyList(); + } + List spanLogs = new ArrayList<>(annotationCount); + for (int i = 0; i < annotationCount; i++) { + Map.Entry entry = Iterators.get(span.getEvents().iterator(), i); + long epochMicros = entry.getKey(); + String value = entry.getValue(); + spanLogs.add(new SpanLog(epochMicros, Collections.singletonMap("annotation", value))); + } + return spanLogs; + } + + // https://github.com/wavefrontHQ/wavefront-opentracing-sdk-java/blob/f1f08d8daf7b692b9b61dcd5bc24ca6befa8e710/src/main/java/com/wavefront/opentracing/WavefrontTracer.java#L275-L280 + static List> createDefaultTags(ApplicationTags applicationTags) { + List> result = new ArrayList<>(); + result.add(Pair.of(APPLICATION_TAG_KEY, applicationTags.getApplication())); + result.add(Pair.of(SERVICE_TAG_KEY, applicationTags.getService())); + result.add(Pair.of(CLUSTER_TAG_KEY, + applicationTags.getCluster() == null ? NULL_TAG_VAL : applicationTags.getCluster())); + result.add( + Pair.of(SHARD_TAG_KEY, applicationTags.getShard() == null ? NULL_TAG_VAL : applicationTags.getShard())); + if (applicationTags.getCustomTags() != null) { + applicationTags.getCustomTags().forEach((k, v) -> result.add(Pair.of(k, v))); + } + return result; + } + + /** + * Extracted for test isolation and as parsing otherwise implies multiple-returns or + * scanning later. + * + *

+ * Ex. {@code SpanDerivedMetricsUtils#reportWavefrontGeneratedData} needs tags + * separately from the component tag and error status. + */ + static final class TagList extends ArrayList> { + + String componentTagValue = NULL_TAG_VAL; + + boolean isError; // See explanation here: + + // https://github.com/openzipkin/brave/pull/1221 + + TagList(Set defaultTagKeys, List> defaultTags, FinishedSpan span) { + super(defaultTags.size() + span.getTags().size()); + // TODO: OTel doesn't have a notion of debug + boolean debug = false; + boolean hasAnnotations = span.getEvents().size() > 0; + isError = span.getError() != null; + + int tagCount = span.getTags().size(); + addAll(defaultTags); + for (int i = 0; i < tagCount; i++) { + String tagKey = Iterators.get(span.getTags().keySet().iterator(), i); + String tagValue = Iterators.get(span.getTags().values().iterator(), i); + String key = tagKey; + String value = tagValue; + String lcKey = key.toLowerCase(Locale.ROOT); + if (lcKey.equals(ERROR_TAG_KEY)) { + isError = true; + continue; // We later replace whatever the potentially empty value was + // with "true" + } + if (value.isEmpty()) { + continue; + } + if (defaultTagKeys.contains(lcKey)) { + continue; + } + if (lcKey.equals(DEBUG_TAG_KEY)) { + debug = true; // This tag is set out-of-band + continue; + } + if (lcKey.equals(COMPONENT_TAG_KEY)) { + componentTagValue = value; + } + add(Pair.of(key, value)); + } + + // Check for span.error() for uncaught exception in request mapping and add it + // to Wavefront span tag + if (isError) { + add(Pair.of("error", "true")); + } + + // https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L300-L303 + if (debug) { + add(Pair.of(DEBUG_TAG_KEY, "true")); + } + + // https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L254-L266 + if (span.getKind() != null) { + String kind = span.getKind().toString().toLowerCase(); + add(Pair.of("span.kind", kind)); + if (hasAnnotations) { + add(Pair.of("_spanSecondaryId", kind)); + } + } + + // https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L329-L332 + if (hasAnnotations) { + add(Pair.of(SPAN_LOG_KEY, "true")); + } + + // https://github.com/wavefrontHQ/wavefront-proxy/blob/3dd1fa11711a04de2d9d418e2269f0f9fb464f36/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java#L324-L327 + if (span.getLocalIp() != null) { + add(Pair.of("ipv4", span.getLocalIp())); // NOTE: this could be IPv6!! + } + } + + } + +} diff --git a/spring-cloud-sleuth-autoconfigure/src/main/resources/META-INF/spring.factories b/spring-cloud-sleuth-autoconfigure/src/main/resources/META-INF/spring.factories index 1c5ba3eba..02e2a0409 100644 --- a/spring-cloud-sleuth-autoconfigure/src/main/resources/META-INF/spring.factories +++ b/spring-cloud-sleuth-autoconfigure/src/main/resources/META-INF/spring.factories @@ -25,7 +25,8 @@ org.springframework.cloud.sleuth.autoconfig.brave.instrument.messaging.BraveMess org.springframework.cloud.sleuth.autoconfig.brave.instrument.opentracing.BraveOpentracingAutoConfiguration,\ org.springframework.cloud.sleuth.autoconfig.brave.instrument.redis.BraveRedisAutoConfiguration,\ org.springframework.cloud.sleuth.autoconfig.brave.instrument.mongodb.BraveMongoDbAutoConfiguration,\ -org.springframework.cloud.sleuth.autoconfig.zipkin2.ZipkinAutoConfiguration +org.springframework.cloud.sleuth.autoconfig.zipkin2.ZipkinAutoConfiguration,\ +org.springframework.cloud.sleuth.autoconfig.wavefront.WavefrontSleuthAutoConfiguration # Environment Post Processor org.springframework.boot.env.EnvironmentPostProcessor=\ org.springframework.cloud.sleuth.autoconfig.TraceEnvironmentPostProcessor,\ diff --git a/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/NoOpWavefrontSender.java b/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/NoOpWavefrontSender.java new file mode 100644 index 000000000..7cb7a50a0 --- /dev/null +++ b/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/NoOpWavefrontSender.java @@ -0,0 +1,78 @@ +/* + * Copyright 2013-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.sleuth.autoconfig; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import com.wavefront.sdk.common.Pair; +import com.wavefront.sdk.common.WavefrontSender; +import com.wavefront.sdk.entities.histograms.HistogramGranularity; +import com.wavefront.sdk.entities.tracing.SpanLog; + +/** + * A noop implementation. Does nothing. + * + * @author Marcin Grzejszczak + * @since 3.0.0 + */ +public class NoOpWavefrontSender implements WavefrontSender { + + @Override + public String getClientId() { + return null; + } + + @Override + public void flush() { + + } + + @Override + public int getFailureCount() { + return 0; + } + + @Override + public void sendDistribution(String name, List> centroids, + Set histogramGranularities, Long timestamp, String source, Map tags) { + + } + + @Override + public void sendMetric(String name, double value, Long timestamp, String source, Map tags) { + + } + + @Override + public void sendFormattedMetric(String point) { + + } + + @Override + public void sendSpan(String name, long startMillis, long durationMillis, String source, UUID traceId, UUID spanId, + List parents, List followsFrom, List> tags, List spanLogs) { + } + + @Override + public void close() { + + } + +} diff --git a/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/TraceNoOpAutoConfiguration.java b/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/TraceNoOpAutoConfiguration.java index 6b94213f1..2c5dc905b 100644 --- a/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/TraceNoOpAutoConfiguration.java +++ b/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/TraceNoOpAutoConfiguration.java @@ -16,6 +16,7 @@ package org.springframework.cloud.sleuth.autoconfig; +import org.springframework.boot.actuate.autoconfigure.metrics.export.wavefront.WavefrontMetricsExportAutoConfiguration; import org.springframework.boot.autoconfigure.AutoConfigureBefore; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -31,6 +32,7 @@ import org.springframework.cloud.sleuth.propagation.Propagator; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; /** * {@link org.springframework.boot.autoconfigure.EnableAutoConfiguration @@ -43,7 +45,7 @@ import org.springframework.context.annotation.Import; */ @Configuration(proxyBeanMethods = false) @ConditionalOnProperty("spring.sleuth.noop.enabled") -@AutoConfigureBefore(BraveAutoConfiguration.class) +@AutoConfigureBefore({ BraveAutoConfiguration.class, WavefrontMetricsExportAutoConfiguration.class }) @Import(TraceConfiguration.class) public class TraceNoOpAutoConfiguration { @@ -68,6 +70,12 @@ public class TraceNoOpAutoConfiguration { return new NoOpSpanCustomizer(); } + @Bean + @Primary + NoOpWavefrontSender noOpWavefrontSender() { + return new NoOpWavefrontSender(); + } + @Configuration(proxyBeanMethods = false) static class TraceHttpConfiguration { diff --git a/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontAutoConfigurationTests.java b/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontAutoConfigurationTests.java new file mode 100644 index 000000000..d814913ca --- /dev/null +++ b/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontAutoConfigurationTests.java @@ -0,0 +1,324 @@ +/* + * Copyright 2013-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.sleuth.autoconfig.wavefront; + +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; + +import brave.Tracer; +import brave.TracingCustomizer; +import brave.handler.SpanHandler; +import com.wavefront.sdk.appagent.jvm.reporter.WavefrontJvmReporter; +import com.wavefront.sdk.common.Pair; +import com.wavefront.sdk.common.WavefrontSender; +import com.wavefront.sdk.common.application.ApplicationTags; +import com.wavefront.spring.autoconfigure.ApplicationTagsBuilderCustomizer; +import com.wavefront.spring.autoconfigure.WavefrontAutoConfiguration; +import io.micrometer.core.instrument.MeterRegistry; +import org.junit.jupiter.api.Test; + +import org.springframework.boot.actuate.autoconfigure.metrics.CompositeMeterRegistryAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.metrics.MetricsAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.metrics.export.simple.SimpleMetricsExportAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.metrics.export.wavefront.WavefrontMetricsExportAutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.FilteredClassLoader; +import org.springframework.boot.test.context.assertj.AssertableApplicationContext; +import org.springframework.boot.test.context.runner.AbstractApplicationContextRunner; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.boot.test.context.runner.ContextConsumer; +import org.springframework.cloud.sleuth.autoconfig.brave.BraveAutoConfiguration; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +/** + * Tests for {@link WavefrontAutoConfiguration}. + * + * @author Stephane Nicoll + * @author Tommy Ludwig + */ +class WavefrontAutoConfigurationTests { + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner().withConfiguration( + AutoConfigurations.of(WavefrontAutoConfiguration.class, WavefrontSleuthAutoConfiguration.class)); + + @Test + void applicationTagsIsConfiguredFromPropertiesWhenNoneExists() { + this.contextRunner + .withPropertyValues("wavefront.application.name=test-app", "wavefront.application.service=test-service") + .run((context) -> { + assertThat(context).hasSingleBean(ApplicationTags.class); + ApplicationTags tags = context.getBean(ApplicationTags.class); + assertThat(tags.getApplication()).isEqualTo("test-app"); + assertThat(tags.getService()).isEqualTo("test-service"); + assertThat(tags.getCluster()).isNull(); + assertThat(tags.getShard()).isNull(); + assertThat(tags.getCustomTags()).isEmpty(); + }); + } + + @Test + void applicationTagsCanBeCustomized() { + this.contextRunner + .withPropertyValues("wavefront.application.name=test-app", "wavefront.application.service=test-service") + .withBean(ApplicationTagsBuilderCustomizer.class, + () -> (builder) -> builder.cluster("test-cluster").shard("test-shard")) + .run((context) -> { + assertThat(context).hasSingleBean(ApplicationTags.class); + ApplicationTags tags = context.getBean(ApplicationTags.class); + assertThat(tags.getApplication()).isEqualTo("test-app"); + assertThat(tags.getService()).isEqualTo("test-service"); + assertThat(tags.getCluster()).isEqualTo("test-cluster"); + assertThat(tags.getShard()).isEqualTo("test-shard"); + assertThat(tags.getCustomTags()).isEmpty(); + }); + } + + @Test + void applicationTagsIsReusedWhenCustomInstanceExists() { + this.contextRunner + .withPropertyValues("wavefront.application.name=test-app", "wavefront.application.service=test-service") + .withBean(ApplicationTags.class, + () -> new ApplicationTags.Builder("another-app", "another-service").build()) + .run((context) -> { + assertThat(context).hasSingleBean(ApplicationTags.class); + ApplicationTags tags = context.getBean(ApplicationTags.class); + assertThat(tags.getApplication()).isEqualTo("another-app"); + assertThat(tags.getService()).isEqualTo("another-service"); + assertThat(tags.getCluster()).isNull(); + assertThat(tags.getShard()).isNull(); + assertThat(tags.getCustomTags()).isEmpty(); + }); + } + + @Test + void applicationTagsAreExportedToWavefrontRegistry() { + this.contextRunner + .withPropertyValues("wavefront.application.name=test-app", "wavefront.application.service=test-service") + .with(wavefrontMetrics(() -> mock(WavefrontSender.class))).run((context) -> { + MeterRegistry registry = context.getBean(MeterRegistry.class); + registry.counter("my.counter", "env", "qa"); + assertThat(registry.find("my.counter").tags("env", "qa").tags("application", "test-app") + .tags("service", "test-service").counter()).isNotNull(); + }); + } + + @Test + void applicationTagsWithFullInformationAreExportedToWavefrontRegistry() { + this.contextRunner + .withPropertyValues("wavefront.application.name=test-app", "wavefront.application.service=test-service", + "wavefront.application.cluster=test-cluster", "wavefront.application.shard=test-shard") + .with(wavefrontMetrics(() -> mock(WavefrontSender.class))).run((context) -> { + MeterRegistry registry = context.getBean(MeterRegistry.class); + registry.counter("my.counter", "env", "qa"); + assertThat(registry.find("my.counter").tags("env", "qa").tags("application", "test-app") + .tags("service", "test-service").tags("cluster", "test-cluster").tags("shard", "test-shard") + .counter()).isNotNull(); + }); + } + + @Test + void applicationTagsAreNotExportedToNonWavefrontRegistry() { + this.contextRunner + .withPropertyValues("wavefront.application.name=test-app", "wavefront.application.service=test-service") + .with(metrics()).withConfiguration(AutoConfigurations.of(SimpleMetricsExportAutoConfiguration.class)) + .run((context) -> { + MeterRegistry registry = context.getBean(MeterRegistry.class); + registry.counter("my.counter", "env", "qa"); + assertThat(registry.find("my.counter").tags("env", "qa")).isNotNull(); + assertThat(registry.find("my.counter").tags("env", "qa").tags("application", "test-app") + .tags("service", "test-service").tags("cluster", "test-cluster").tags("shard", "test-shard") + .counter()).isNull(); + }); + } + + @Test + void jvmReporterIsConfiguredWhenNoneExists() { + this.contextRunner.with(wavefrontMetrics(() -> mock(WavefrontSender.class))) + .run((context) -> assertThat(context).hasSingleBean(WavefrontJvmReporter.class)); + } + + @Test + void jvmReporterCanBeDisabled() { + this.contextRunner.withPropertyValues("wavefront.metrics.extract-jvm-metrics=false") + .with(wavefrontMetrics(() -> mock(WavefrontSender.class))) + .run(context -> assertThat(context).doesNotHaveBean(WavefrontJvmReporter.class)); + } + + @Test + void jvmReporterCanBeCustomized() { + WavefrontJvmReporter reporter = mock(WavefrontJvmReporter.class); + this.contextRunner.with(wavefrontMetrics(() -> mock(WavefrontSender.class))) + .withBean(WavefrontJvmReporter.class, () -> reporter) + .run((context) -> assertThat(context).getBean(WavefrontJvmReporter.class).isEqualTo(reporter)); + } + + @Test + void jvmReporterNotConfiguredWithoutWavefrontSender() { + this.contextRunner.with(metrics()) + .run(context -> assertThat(context).doesNotHaveBean(WavefrontJvmReporter.class)); + } + + @Test + void tracingWithSleuthIsConfiguredWithWavefrontSender() { + WavefrontSender sender = mock(WavefrontSender.class); + this.contextRunner.withPropertyValues().with(wavefrontMetrics(() -> sender)).with(sleuth()).run((context) -> { + assertThat(context).hasSingleBean(TracingCustomizer.class); + WavefrontSleuthBraveSpanHandler braveSpanHandler = extractSpanHandler(context.getBean(Tracer.class)); + assertThat(braveSpanHandler.spanHandler).hasFieldOrPropertyWithValue("wavefrontSender", sender); + }); + } + + @Test + void tracingWithSleuthWithEmptyEnvironmentUseDefaultTags() { + this.contextRunner.with(wavefrontMetrics(() -> mock(WavefrontSender.class))).with(sleuth()) + .run(assertSleuthSpanDefaultTags("unnamed_application", "unnamed_service")); + } + + @Test + void tracingWithSleuthWithWavefrontTagsAndSpringApplicationNameUseWavefrontTags() { + this.contextRunner + .withPropertyValues("wavefront.application.name=wavefront-application", + "wavefront.application.service=wavefront-service", "spring.application.name=spring-service") + .with(wavefrontMetrics(() -> mock(WavefrontSender.class))).with(sleuth()) + .run(assertSleuthSpanDefaultTags("wavefront-application", "wavefront-service")); + } + + @Test + void tracingWithSleuthWithSpringApplicationNameUseItRatherThanDefault() { + this.contextRunner.withPropertyValues("spring.application.name=spring-service") + .with(wavefrontMetrics(() -> mock(WavefrontSender.class))).with(sleuth()) + .run(assertSleuthSpanDefaultTags("unnamed_application", "spring-service")); + } + + @Test + void tracingWithSleuthWithCustomApplicationTagsUseThat() { + this.contextRunner + .withPropertyValues("wavefront.application.name=wavefront-application", + "wavefront.application.service=wavefront-service") + .with(wavefrontMetrics(() -> mock(WavefrontSender.class))) + .withBean(ApplicationTags.class, + () -> new ApplicationTags.Builder("custom-application", "custom-service") + .cluster("custom-cluster").shard("custom-shard").build()) + .with(sleuth()).run(assertSleuthSpanDefaultTags("custom-application", "custom-service", + "custom-cluster", "custom-shard")); + } + + @Test + void tracingWithSleuthWithCustomApplicationTagsAndEmptyValuesFallbackToDefaults() { + this.contextRunner + .withPropertyValues("wavefront.application.name=wavefront-application", + "wavefront.application.service=wavefront-service") + .with(wavefrontMetrics(() -> mock(WavefrontSender.class))) + .withBean(ApplicationTags.class, + () -> new ApplicationTags.Builder("custom-application", "custom-service").build()) + .with(sleuth()) + .run(assertSleuthSpanDefaultTags("custom-application", "custom-service", "none", "none")); + } + + private ContextConsumer assertSleuthSpanDefaultTags(String applicationName, + String serviceName) { + return assertSleuthSpanDefaultTags(applicationName, serviceName, "none", "none"); + } + + private ContextConsumer assertSleuthSpanDefaultTags(String applicationName, + String serviceName, String cluster, String shard) { + return (context) -> { + assertThat(context).hasSingleBean(TracingCustomizer.class); + WavefrontSleuthBraveSpanHandler braveSpanHandler = extractSpanHandler(context.getBean(Tracer.class)); + assertThat(braveSpanHandler.spanHandler.getDefaultTags()).contains( + new Pair<>("application", applicationName), new Pair<>("service", serviceName), + new Pair<>("cluster", cluster), new Pair<>("shard", shard)); + }; + } + + @SuppressWarnings("unchecked") + @Test + void tracingWithSleuthCanBeConfigured() { + WavefrontSender sender = mock(WavefrontSender.class); + this.contextRunner.withPropertyValues() + .withPropertyValues("wavefront.tracing.red-metrics-custom-tag-keys=region,test") + .with(wavefrontMetrics(() -> sender)).with(sleuth()).run((context) -> { + assertThat(context).hasSingleBean(TracingCustomizer.class); + WavefrontSleuthBraveSpanHandler braveSpanHandler = extractSpanHandler( + context.getBean(Tracer.class)); + WavefrontSleuthSpanHandler spanHandler = braveSpanHandler.spanHandler; + Set traceDerivedCustomTagKeys = (Set) ReflectionTestUtils.getField(spanHandler, + "traceDerivedCustomTagKeys"); + assertThat(traceDerivedCustomTagKeys).containsExactlyInAnyOrder("region", "test"); + }); + } + + @Test + void tracingWithOpenTracingBacksOffWhenSpringCloudSleuthIsAvailable() { + this.contextRunner.with(wavefrontMetrics(() -> mock(WavefrontSender.class))) + .run((context) -> assertThat(context).hasSingleBean(TracingCustomizer.class) + .doesNotHaveBean(io.opentracing.Tracer.class)); + } + + @Test + void tracingIsDisabledWhenOpenTracingAndSleuthAreNotAvailable() { + new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(WavefrontAutoConfiguration.class)) + .withClassLoader(new FilteredClassLoader("org.springframework.cloud.sleuth", "io.opentracing")) + .with(wavefrontMetrics(() -> mock(WavefrontSender.class))).run((context) -> assertThat(context) + .doesNotHaveBean(TracingCustomizer.class).doesNotHaveBean(io.opentracing.Tracer.class)); + } + + @Test + void tracingCanBeDisabled() { + this.contextRunner.withPropertyValues("wavefront.tracing.enabled=false") + .with(wavefrontMetrics(() -> mock(WavefrontSender.class))).run((context) -> assertThat(context) + .doesNotHaveBean(TracingCustomizer.class).doesNotHaveBean(io.opentracing.Tracer.class)); + } + + @Test + void tracingIsNotConfiguredWithNonWavefrontRegistry() { + this.contextRunner.with(metrics()).run((context) -> assertThat(context).doesNotHaveBean(Tracer.class)); + } + + @SuppressWarnings("ConstantConditions") + private WavefrontSleuthBraveSpanHandler extractSpanHandler(Tracer tracer) { + SpanHandler[] handlers = (SpanHandler[]) ReflectionTestUtils.getField( + ReflectionTestUtils.getField(ReflectionTestUtils.getField(tracer, "spanHandler"), "delegate"), + "handlers"); + return (WavefrontSleuthBraveSpanHandler) handlers[1]; + } + + @SuppressWarnings("unchecked") + private static > Function wavefrontMetrics( + Supplier wavefrontSender) { + return (runner) -> (T) runner.withBean(WavefrontSender.class, wavefrontSender) + .withConfiguration(AutoConfigurations.of(WavefrontMetricsExportAutoConfiguration.class)) + .with(metrics()); + } + + @SuppressWarnings("unchecked") + private static > Function metrics() { + return (runner) -> (T) runner.withPropertyValues("management.metrics.use-global-registry=false") + .withConfiguration(AutoConfigurations.of(MetricsAutoConfiguration.class, + CompositeMeterRegistryAutoConfiguration.class)); + } + + @SuppressWarnings("unchecked") + private static > Function sleuth() { + return (runner) -> (T) runner.withConfiguration(AutoConfigurations.of(BraveAutoConfiguration.class)); + } + +} diff --git a/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontTracingIntegrationTests.java b/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontTracingIntegrationTests.java new file mode 100644 index 000000000..9f575bf99 --- /dev/null +++ b/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/wavefront/WavefrontTracingIntegrationTests.java @@ -0,0 +1,369 @@ +/* + * Copyright 2013-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.sleuth.autoconfig.wavefront; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; + +import brave.Tracing; +import brave.internal.Platform; +import brave.opentracing.BraveTracer; +import brave.sampler.Sampler; +import com.wavefront.sdk.common.Pair; +import com.wavefront.sdk.common.WavefrontSender; +import com.wavefront.sdk.entities.histograms.HistogramGranularity; +import com.wavefront.sdk.entities.tracing.SpanLog; +import io.opentracing.Tracer; +import io.opentracing.tag.Tags; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.autoconfigure.actuate.metrics.AutoConfigureMetrics; +import org.springframework.boot.test.autoconfigure.web.reactive.AutoConfigureWebTestClient; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.web.reactive.server.WebTestClient; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Integration tests for tracing. + */ +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = WavefrontTracingIntegrationTests.Config.class, + properties = { "wavefront.application.name=IntegratedTracingTests", "spring.application.name=test_service" }) +@AutoConfigureWebTestClient +@AutoConfigureMetrics +@DirtiesContext +public class WavefrontTracingIntegrationTests { + + @Autowired + private WebTestClient client; + + @Autowired + private BlockingDeque spanRecordQueue; + + @Test + void sendsToWavefront() { + this.client.get().uri("/api/fn/10").header("b3", "0000000000000001-0000000000000003-1-0000000000000002") + .exchange().expectStatus().isOk(); + + SpanRecord spanRecord = takeRecord(spanRecordQueue); + assertThat(spanRecord.traceId).hasToString("00000000-0000-0000-0000-000000000001"); + assertThat(spanRecord.parents).extracting(UUID::toString) + .containsExactly("00000000-0000-0000-0000-000000000003"); + assertThat(spanRecord.followsFrom).isNull(); + // This tests that RPC spans do not share the same span ID + assertThat(spanRecord.spanId.toString()).isNotEqualTo("00000000-0000-0000-0000-000000000003") + .matches("^[0-9a-f]{8}\\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\\b[0-9a-f]{12}$"); + assertThat(spanRecord.name).isEqualTo("GET /api/fn/{id}"); + + // spot check the unit is valid (millis not micros) + long currentTime = System.currentTimeMillis(); + assertThat(spanRecord.startMillis).isGreaterThan(currentTime - 5000).isLessThan(currentTime); + // Less than a millis should round up to 1, but the test could take longer than + // 1ms + assertThat(spanRecord.durationMillis).isPositive(); + + // http + assertThat(spanRecord.tags).containsExactlyInAnyOrder(Pair.of("application", "IntegratedTracingTests"), + Pair.of("service", "test_service"), Pair.of("cluster", "none"), Pair.of("shard", "none"), + Pair.of("http.method", "GET"), Pair.of("http.path", "/api/fn/10"), + Pair.of("mvc.controller.class", "WebMvcController"), Pair.of("mvc.controller.method", "fn"), + Pair.of("span.kind", "server"), Pair.of("ipv4", Platform.get().linkLocalIp())); + } + + @Test + void http_badRequest_setsStatusCodeAndErrorTrueTags() { + this.client.get().uri("/badrequest").exchange().expectStatus().isBadRequest(); + + SpanRecord spanRecord = takeRecord(spanRecordQueue); + + // http + assertThat(spanRecord.tags).contains(Pair.of("http.status_code", "400"), Pair.of("error", "true")); + } + + @Test + void setsStatusCodeAndErrorTrueTags_opentracing() { + this.client.get().uri("/error/opentracing").exchange().expectStatus().is5xxServerError(); + + SpanRecord spanRecord = takeRecord(spanRecordQueue); + // http + assertThat(spanRecord.tags).contains(Pair.of("http.status_code", "500"), Pair.of("error", "true") // retains + // the + // boolean + // true + ); + } + + @Test + void setsStatusCodeAndErrorTrueTags_brave() { + this.client.get().uri("/error/brave").exchange().expectStatus().is5xxServerError(); + + SpanRecord spanRecord = takeRecord(spanRecordQueue); + // http + assertThat(spanRecord.tags).contains(Pair.of("http.status_code", "500"), Pair.of("error", "true") // deletes + // the + // user + // message + ); + } + + @Test + void setsStatusCodeAndErrorTrueTags_exception() { + this.client.get().uri("/error/exception").exchange().expectStatus().is5xxServerError(); + + // http + assertThat(takeRecord(spanRecordQueue).tags).contains(Pair.of("http.status_code", "500"), + Pair.of("error", "true") // deletes the exception message + ); + } + + @Test + void setsStatusCodeAndErrorTrueTags_thrownException() { + this.client.get().uri("/throws").exchange().expectStatus().is5xxServerError(); + + SpanRecord spanRecord = takeRecord(spanRecordQueue); + // http + assertThat(spanRecord.tags).contains( + /* Pair.of("http.status_code", "500"), */ // Able to return error=true + // span tag but not + // http.status_code span tag + Pair.of("error", "true") // deletes the exception message + ); + } + + /** Helps ensure test bugs don't result in hung tests! */ + R takeRecord(BlockingDeque queue) { + R result; + try { + result = queue.poll(3, TimeUnit.SECONDS); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + + assertThat(result).withFailMessage("Record was not reported").isNotNull(); + return result; + } + + /** Makes sure tests aren't accidentally not verifying all reported data. */ + @AfterEach + void ensureNoExtraSpans() { + try { + SpanRecord span = spanRecordQueue.poll(100, TimeUnit.MILLISECONDS); + assertThat(span).withFailMessage("Span remaining in queue. Check for redundant reporting: %s", span) + .isNull(); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } + + @Configuration + @EnableAutoConfiguration + static class Config { + + /** + * This uses a {@linkplain Controller WebMVC controller} as it is the most popular + * way to write Spring services and has no instrumentation gotchas or scope bugs + * like reactive tracing. This allows us to focus on api and data mapping issues, + * which is the heart of this test. + */ + @Bean + WebMvcController controller() { + return new WebMvcController(); + } + + @Bean + Sampler sampler() { + return Sampler.ALWAYS_SAMPLE; + } + + /** + * Sleuth would automatically wire this, except there's another impl in the + * classpath. + */ + @Bean + Tracer opentracing(Tracing tracing) { + return BraveTracer.create(tracing); + } + + @Bean + BlockingDeque spanRecordQueue() { + return new LinkedBlockingDeque<>(); + } + + @Bean + @Primary + WavefrontSender wavefrontSender(BlockingDeque spanRecordQueue) { + return new WavefrontSender() { + @Override + public String getClientId() { + return null; + } + + @Override + public void flush() { + + } + + @Override + public int getFailureCount() { + return 0; + } + + @Override + public void sendDistribution(String name, List> centroids, + Set histogramGranularities, Long timestamp, String source, + Map tags) { + + } + + @Override + public void sendMetric(String name, double value, Long timestamp, String source, + Map tags) { + + } + + @Override + public void sendFormattedMetric(String point) { + + } + + @Override + public void sendSpan(String name, long startMillis, long durationMillis, String source, UUID traceId, + UUID spanId, List parents, List followsFrom, List> tags, + List spanLogs) { + spanRecordQueue.add(new SpanRecord(name, startMillis, durationMillis, source, traceId, spanId, + parents, followsFrom, tags, spanLogs)); + } + + @Override + public void close() { + + } + }; + } + + } + + @Controller + static class WebMvcController { + + @Autowired + Tracer opentracing; + + @Autowired + brave.Tracer tracer; + + @RequestMapping("/api/fn/{id}") + public ResponseEntity fn(@PathVariable("id") String id) { + return new ResponseEntity<>(id, HttpStatus.OK); + } + + @RequestMapping("/error/{api}") + public ResponseEntity error(@PathVariable("api") String api) { + switch (api) { + case "brave": + tracer.currentSpanCustomizer().tag("error", "user message"); + break; + case "opentracing": + opentracing.activeSpan().setTag(Tags.ERROR, true); + break; + case "exception": + tracer.currentSpan().error(new RuntimeException("uncaught!")); + break; + default: + return new ResponseEntity<>(HttpStatus.BAD_REQUEST); + } + return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); + } + + @RequestMapping("/badrequest") + public ResponseEntity badrequest() { + return new ResponseEntity<>(HttpStatus.BAD_REQUEST); + } + + @RequestMapping("throws") + public void toss() { + throw new IllegalStateException("boom"); + } + + } + + static final class SpanRecord { + + private final String name; + + private final long startMillis; + + private final long durationMillis; + + private final String source; + + private final UUID traceId; + + private final UUID spanId; + + private final List parents; + + private final List followsFrom; + + private final List> tags; + + private final List spanLogs; + + SpanRecord(String name, long startMillis, long durationMillis, String source, UUID traceId, UUID spanId, + List parents, List followsFrom, List> tags, List spanLogs) { + this.name = name; + this.startMillis = startMillis; + this.durationMillis = durationMillis; + this.source = source; + this.traceId = traceId; + this.spanId = spanId; + this.parents = parents; + this.followsFrom = followsFrom; + this.tags = tags; + this.spanLogs = spanLogs; + } + + @Override + public String toString() { + return "SpanRecord{" + "name='" + name + '\'' + ", traceId=" + traceId + ", spanId=" + spanId + '}'; + } + + } + +} diff --git a/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/zipkin2/ZipkinSamplerTests.java b/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/zipkin2/ZipkinSamplerTests.java index ec6546553..40a92cc1f 100644 --- a/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/zipkin2/ZipkinSamplerTests.java +++ b/spring-cloud-sleuth-autoconfigure/src/test/java/org/springframework/cloud/sleuth/autoconfig/zipkin2/ZipkinSamplerTests.java @@ -29,6 +29,8 @@ import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.cloud.gateway.config.GatewayAutoConfiguration; import org.springframework.cloud.gateway.config.GatewayClassPathWarningAutoConfiguration; import org.springframework.cloud.gateway.config.GatewayMetricsAutoConfiguration; +import org.springframework.cloud.sleuth.autoconfig.NoOpWavefrontSender; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; /** @@ -56,6 +58,11 @@ public class ZipkinSamplerTests { MongoAutoConfiguration.class, QuartzAutoConfiguration.class }) static class TestConfig { + @Bean + NoOpWavefrontSender noOpWavefrontSender() { + return new NoOpWavefrontSender(); + } + } } diff --git a/spring-cloud-sleuth-autoconfigure/src/test/resources/application.yml b/spring-cloud-sleuth-autoconfigure/src/test/resources/application.yml index 7712f8c1c..b2e56d8f2 100644 --- a/spring-cloud-sleuth-autoconfigure/src/test/resources/application.yml +++ b/spring-cloud-sleuth-autoconfigure/src/test/resources/application.yml @@ -1,3 +1,7 @@ logging.level.org.springframework.cloud: DEBUG spring.autoconfigure.exclude: org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration, org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration, org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration, org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration, org.springframework.cloud.gateway.config.GatewayAutoConfiguration, org.springframework.cloud.gateway.config.GatewayClassPathWarningAutoConfiguration, org.springframework.cloud.gateway.config.GatewayMetricsAutoConfiguration, org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration + +wavefront: + metrics: + extract-jvm-metrics: false \ No newline at end of file