489 lines
19 KiB
HTML
489 lines
19 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<!--[if IE]><meta http-equiv="X-UA-Compatible" content="IE=edge"><![endif]-->
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<meta name="generator" content="Asciidoctor 1.5.8">
|
|
<title>Features</title>
|
|
<link rel="stylesheet" href="css/spring.css">
|
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
|
|
|
|
<style>
|
|
.hidden {
|
|
display: none;
|
|
}
|
|
|
|
.switch {
|
|
border-width: 1px 1px 0 1px;
|
|
border-style: solid;
|
|
border-color: #7a2518;
|
|
display: inline-block;
|
|
}
|
|
|
|
.switch--item {
|
|
padding: 10px;
|
|
background-color: #ffffff;
|
|
color: #7a2518;
|
|
display: inline-block;
|
|
cursor: pointer;
|
|
}
|
|
|
|
.switch--item:not(:first-child) {
|
|
border-width: 0 0 0 1px;
|
|
border-style: solid;
|
|
border-color: #7a2518;
|
|
}
|
|
|
|
.switch--item.selected {
|
|
background-color: #7a2519;
|
|
color: #ffffff;
|
|
}
|
|
</style>
|
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/zepto/1.2.0/zepto.min.js"></script>
|
|
<script type="text/javascript">
|
|
function addBlockSwitches() {
|
|
$('.primary').each(function() {
|
|
primary = $(this);
|
|
createSwitchItem(primary, createBlockSwitch(primary)).item.addClass("selected");
|
|
primary.children('.title').remove();
|
|
});
|
|
$('.secondary').each(function(idx, node) {
|
|
secondary = $(node);
|
|
primary = findPrimary(secondary);
|
|
switchItem = createSwitchItem(secondary, primary.children('.switch'));
|
|
switchItem.content.addClass('hidden');
|
|
findPrimary(secondary).append(switchItem.content);
|
|
secondary.remove();
|
|
});
|
|
}
|
|
|
|
function createBlockSwitch(primary) {
|
|
blockSwitch = $('<div class="switch"></div>');
|
|
primary.prepend(blockSwitch);
|
|
return blockSwitch;
|
|
}
|
|
|
|
function findPrimary(secondary) {
|
|
candidate = secondary.prev();
|
|
while (!candidate.is('.primary')) {
|
|
candidate = candidate.prev();
|
|
}
|
|
return candidate;
|
|
}
|
|
|
|
function createSwitchItem(block, blockSwitch) {
|
|
blockName = block.children('.title').text();
|
|
content = block.children('.content').first().append(block.next('.colist'));
|
|
item = $('<div class="switch--item">' + blockName + '</div>');
|
|
item.on('click', '', content, function(e) {
|
|
$(this).addClass('selected');
|
|
$(this).siblings().removeClass('selected');
|
|
e.data.siblings('.content').addClass('hidden');
|
|
e.data.removeClass('hidden');
|
|
});
|
|
blockSwitch.append(item);
|
|
return {'item': item, 'content': content};
|
|
}
|
|
|
|
function globalSwitch() {
|
|
$('.switch--item').each(function() {
|
|
$(this).off('click');
|
|
$(this).on('click', function() {
|
|
selectedText = $(this).text()
|
|
selectedIndex = $(this).index()
|
|
$(".switch--item").filter(function() { return ($(this).text() === selectedText) }).each(function() {
|
|
$(this).addClass('selected');
|
|
$(this).siblings().removeClass('selected');
|
|
selectedContent = $(this).parent().siblings(".content").eq(selectedIndex)
|
|
selectedContent.removeClass('hidden');
|
|
selectedContent.siblings().addClass('hidden');
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
$(addBlockSwitches);
|
|
$(globalSwitch);
|
|
|
|
</script>
|
|
|
|
</head>
|
|
<body class="book toc2 toc-left">
|
|
<div id="header">
|
|
<div id="toc" class="toc2">
|
|
<div id="toctitle">Table of Contents</div>
|
|
<ul class="sectlevel1">
|
|
<li><a href="#features">1. Features</a>
|
|
<ul class="sectlevel2">
|
|
<li><a href="#contextualizing-errors">1.1. Contextualizing errors</a></li>
|
|
<li><a href="#log-correlation">1.2. Log correlation</a>
|
|
<ul class="sectlevel3">
|
|
<li><a href="#json-logback-with-logstash">1.2.1. JSON Logback with Logstash</a></li>
|
|
</ul>
|
|
</li>
|
|
<li><a href="#service-dependency-graph">1.3. Service Dependency Graph</a></li>
|
|
<li><a href="#request-scoped-properties-baggage">1.4. Request scoped properties (Baggage)</a>
|
|
<ul class="sectlevel3">
|
|
<li><a href="#baggage-versus-tags">1.4.1. Baggage versus Tags</a></li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
<div id="content">
|
|
<div class="sect1">
|
|
<h2 id="features"><a class="anchor" href="#features"></a><a class="link" href="#features">1. Features</a></h2>
|
|
<div class="sectionbody">
|
|
<div class="paragraph">
|
|
<p>Sleuth sets up instrumentation not only to track timing, but also to catch
|
|
errors so that they can be analyzed or correlated with logs. This works the
|
|
same way regardless of if the error came from a common instrumented library,
|
|
such as <code>RestTemplate</code>, or your own code annotated with <code>@NewSpan</code> or similar.</p>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>Below, we’ll use the word Zipkin to describe the tracing system, and include
|
|
Zipkin screenshots. However, most services accepting <a href="https://zipkin.io/zipkin-api/#/default/post_spans">Zipkin format</a>
|
|
have similar base features. Sleuth can also be configured to send data in other
|
|
formats, something detailed later.</p>
|
|
</div>
|
|
<div class="sect2">
|
|
<h3 id="contextualizing-errors"><a class="anchor" href="#contextualizing-errors"></a><a class="link" href="#contextualizing-errors">1.1. Contextualizing errors</a></h3>
|
|
<div class="paragraph">
|
|
<p>Without distributed tracing, it can be difficult to understand the impact of a
|
|
an exception. For example, it can be hard to know if a specific request caused
|
|
the caller to fail or not.</p>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>Zipkin reduces time in triage by contextualizing errors and delays.</p>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>Requests colored red in the search screen failed:</p>
|
|
</div>
|
|
<div class="imageblock">
|
|
<div class="content">
|
|
<img src="https://raw.githubusercontent.com/spring-cloud/spring-cloud-sleuth/master/docs/src/main/asciidoc/images/zipkin-error-traces.png" alt="Error Traces">
|
|
</div>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>If you then click on one of the traces, you can understand if the failure
|
|
happened before the request hit another service or not:</p>
|
|
</div>
|
|
<div class="imageblock">
|
|
<div class="content">
|
|
<img src="https://raw.githubusercontent.com/spring-cloud/spring-cloud-sleuth/master/docs/src/main/asciidoc/images/zipkin-error-trace-screenshot.png" alt="Error Traces Info propagation">
|
|
</div>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>For example, the above error happened in the "backend" service, and caused the
|
|
"frontend" service to fail.</p>
|
|
</div>
|
|
</div>
|
|
<div class="sect2">
|
|
<h3 id="log-correlation"><a class="anchor" href="#log-correlation"></a><a class="link" href="#log-correlation">1.2. Log correlation</a></h3>
|
|
<div class="paragraph">
|
|
<p>Sleuth configures the logging context with variables including the service name
|
|
(<code>%{spring.zipkin.service.name}</code>) and the trace ID (<code>%{traceId}</code>). These help
|
|
you connect logs with distributed traces and allow you choice in what tools you
|
|
use to troubleshoot your services.</p>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>Once you find any log with an error, you can look for the trace ID in the
|
|
message. Paste that into Zipkin to visualize the entire trace, regardless of
|
|
how many services the first request ended up hitting.</p>
|
|
</div>
|
|
<div class="listingblock">
|
|
<div class="content">
|
|
<pre class="highlightjs highlight"><code>backend.log: 2020-04-09 17:45:40.516 ERROR [backend,5e8eeec48b08e26882aba313eb08f0a4,dcc1df555b5777b3,true] 97203 --- [nio-9000-exec-1] o.s.c.s.i.web.ExceptionLoggingFilter : Uncaught exception thrown
|
|
frontend.log:2020-04-09 17:45:40.574 ERROR [frontend,5e8eeec48b08e26882aba313eb08f0a4,82aba313eb08f0a4,true] 97192 --- [nio-8081-exec-2] o.s.c.s.i.web.ExceptionLoggingFilter : Uncaught exception thrown</code></pre>
|
|
</div>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>Above, you’ll notice the trace ID is <code>5e8eeec48b08e26882aba313eb08f0a4</code>, for
|
|
example. This log configuration was automatically setup by Sleuth.</p>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>If you use a log aggregating tool (such as <a href="https://www.elastic.co/products/kibana">Kibana</a>, <a href="https://www.splunk.com/">Splunk</a>, and others), you can order the events that took place.
|
|
An example from Kibana would resemble the following image:</p>
|
|
</div>
|
|
<div class="imageblock">
|
|
<div class="content">
|
|
<img src="https://raw.githubusercontent.com/spring-cloud/spring-cloud-sleuth/master/docs/src/main/asciidoc/images/kibana.png" alt="Log correlation with Kibana">
|
|
</div>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>If you want to use <a href="https://www.elastic.co/guide/en/logstash/current/index.html">Logstash</a>, the following listing shows the Grok pattern for Logstash:</p>
|
|
</div>
|
|
<div class="listingblock">
|
|
<div class="content">
|
|
<pre class="highlightjs highlight"><code>filter {
|
|
# pattern matching logback pattern
|
|
grok {
|
|
match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\s+%{LOGLEVEL:severity}\s+\[%{DATA:service},%{DATA:trace},%{DATA:span}\]\s+%{DATA:pid}\s+---\s+\[%{DATA:thread}\]\s+%{DATA:class}\s+:\s+%{GREEDYDATA:rest}" }
|
|
}
|
|
date {
|
|
match => ["timestamp", "ISO8601"]
|
|
}
|
|
mutate {
|
|
remove_field => ["timestamp"]
|
|
}
|
|
}</code></pre>
|
|
</div>
|
|
</div>
|
|
<div class="admonitionblock note">
|
|
<table>
|
|
<tr>
|
|
<td class="icon">
|
|
<i class="fa icon-note" title="Note"></i>
|
|
</td>
|
|
<td class="content">
|
|
If you want to use Grok together with the logs from Cloud Foundry, you have to use the following pattern:
|
|
</td>
|
|
</tr>
|
|
</table>
|
|
</div>
|
|
<div class="listingblock">
|
|
<div class="content">
|
|
<pre class="highlightjs highlight"><code>filter {
|
|
# pattern matching logback pattern
|
|
grok {
|
|
match => { "message" => "(?m)OUT\s+%{TIMESTAMP_ISO8601:timestamp}\s+%{LOGLEVEL:severity}\s+\[%{DATA:service},%{DATA:trace},%{DATA:span}\]\s+%{DATA:pid}\s+---\s+\[%{DATA:thread}\]\s+%{DATA:class}\s+:\s+%{GREEDYDATA:rest}" }
|
|
}
|
|
date {
|
|
match => ["timestamp", "ISO8601"]
|
|
}
|
|
mutate {
|
|
remove_field => ["timestamp"]
|
|
}
|
|
}</code></pre>
|
|
</div>
|
|
</div>
|
|
<div class="sect3">
|
|
<h4 id="json-logback-with-logstash"><a class="anchor" href="#json-logback-with-logstash"></a><a class="link" href="#json-logback-with-logstash">1.2.1. JSON Logback with Logstash</a></h4>
|
|
<div class="paragraph">
|
|
<p>Often, you do not want to store your logs in a text file but in a JSON file that Logstash can immediately pick.
|
|
To do so, you have to do the following (for readability, we pass the dependencies in the <code>groupId:artifactId:version</code> notation).</p>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p><strong>Dependencies Setup</strong></p>
|
|
</div>
|
|
<div class="olist arabic">
|
|
<ol class="arabic">
|
|
<li>
|
|
<p>Ensure that Logback is on the classpath (<code>ch.qos.logback:logback-core</code>).</p>
|
|
</li>
|
|
<li>
|
|
<p>Add Logstash Logback encode. For example, to use version <code>4.6</code>, add <code>net.logstash.logback:logstash-logback-encoder:4.6</code>.</p>
|
|
</li>
|
|
</ol>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p><strong>Logback Setup</strong></p>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>Consider the following example of a Logback configuration file (logback-spring.xml).</p>
|
|
</div>
|
|
<div class="listingblock">
|
|
<div class="content">
|
|
<pre class="highlightjs highlight"><code class="language-xml hljs" data-lang="xml"><?xml version="1.0" encoding="UTF-8"?>
|
|
<configuration>
|
|
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
|
<springProperty scope="context" name="springAppName" source="spring.application.name"/>
|
|
<!-- Example for logging into the build folder of your project -->
|
|
<property name="LOG_FILE" value="${BUILD_FOLDER:-build}/${springAppName}"/>
|
|
|
|
<!-- You can override this to have a custom pattern -->
|
|
<property name="CONSOLE_LOG_PATTERN"
|
|
value="%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}"/>
|
|
|
|
<!-- Appender to log to console -->
|
|
<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
|
|
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
|
|
<!-- Minimum logging level to be presented in the console logs-->
|
|
<level>DEBUG</level>
|
|
</filter>
|
|
<encoder>
|
|
<pattern>${CONSOLE_LOG_PATTERN}</pattern>
|
|
<charset>utf8</charset>
|
|
</encoder>
|
|
</appender>
|
|
|
|
<!-- Appender to log to file -->
|
|
<appender name="flatfile" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
|
<file>${LOG_FILE}</file>
|
|
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
|
<fileNamePattern>${LOG_FILE}.%d{yyyy-MM-dd}.gz</fileNamePattern>
|
|
<maxHistory>7</maxHistory>
|
|
</rollingPolicy>
|
|
<encoder>
|
|
<pattern>${CONSOLE_LOG_PATTERN}</pattern>
|
|
<charset>utf8</charset>
|
|
</encoder>
|
|
</appender>
|
|
<!-- Appender to log to file in a JSON format -->
|
|
<appender name="logstash" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
|
<file>${LOG_FILE}.json</file>
|
|
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
|
<fileNamePattern>${LOG_FILE}.json.%d{yyyy-MM-dd}.gz</fileNamePattern>
|
|
<maxHistory>7</maxHistory>
|
|
</rollingPolicy>
|
|
<encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
|
|
<providers>
|
|
<timestamp>
|
|
<timeZone>UTC</timeZone>
|
|
</timestamp>
|
|
<pattern>
|
|
<pattern>
|
|
{
|
|
"timestamp": "@timestamp",
|
|
"severity": "%level",
|
|
"service": "${springAppName:-}",
|
|
"trace": "%X{traceId:-}",
|
|
"span": "%X{spanId:-}",
|
|
"pid": "${PID:-}",
|
|
"thread": "%thread",
|
|
"class": "%logger{40}",
|
|
"rest": "%message"
|
|
}
|
|
</pattern>
|
|
</pattern>
|
|
</providers>
|
|
</encoder>
|
|
</appender>
|
|
<root level="INFO">
|
|
<appender-ref ref="console"/>
|
|
<!-- uncomment this to have also JSON logs -->
|
|
<!--<appender-ref ref="logstash"/>-->
|
|
<!--<appender-ref ref="flatfile"/>-->
|
|
</root>
|
|
</configuration></code></pre>
|
|
</div>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>That Logback configuration file:</p>
|
|
</div>
|
|
<div class="ulist">
|
|
<ul>
|
|
<li>
|
|
<p>Logs information from the application in a JSON format to a <code>build/${spring.application.name}.json</code> file.</p>
|
|
</li>
|
|
<li>
|
|
<p>Has commented out two additional appenders: console and standard log file.</p>
|
|
</li>
|
|
<li>
|
|
<p>Has the same logging pattern as the one presented in the previous section.</p>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
<div class="admonitionblock note">
|
|
<table>
|
|
<tr>
|
|
<td class="icon">
|
|
<i class="fa icon-note" title="Note"></i>
|
|
</td>
|
|
<td class="content">
|
|
If you use a custom <code>logback-spring.xml</code>, you must pass the <code>spring.application.name</code> in the <code>bootstrap</code> rather than the <code>application</code> property file.
|
|
Otherwise, your custom logback file does not properly read the property.
|
|
</td>
|
|
</tr>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="sect2">
|
|
<h3 id="service-dependency-graph"><a class="anchor" href="#service-dependency-graph"></a><a class="link" href="#service-dependency-graph">1.3. Service Dependency Graph</a></h3>
|
|
<div class="paragraph">
|
|
<p>When you consider distributed tracing tracks requests, it makes sense that
|
|
trace data can paint a picture of your architecture.</p>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>Zipkin includes a tool to build service dependency diagrams from traces,
|
|
including the count of calls and how many errors exist.</p>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>The example application will make a simple diagram like this, but your real
|
|
environment diagram may be more complex.
|
|
image::https://raw.githubusercontent.com/spring-cloud/spring-cloud-sleuth/master/docs/src/main/asciidoc/images/zipkin-depedendencies.png[Zipkin Dependencies]</p>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p><strong>Note</strong>: Production environments will generate a lot of data. You will likely
|
|
need to run a separate service to aggregate the dependency graph. You can learn
|
|
more <a href="https://github.com/openzipkin/zipkin-dependencies/">here</a>.</p>
|
|
</div>
|
|
</div>
|
|
<div class="sect2">
|
|
<h3 id="request-scoped-properties-baggage"><a class="anchor" href="#request-scoped-properties-baggage"></a><a class="link" href="#request-scoped-properties-baggage">1.4. Request scoped properties (Baggage)</a></h3>
|
|
<div class="paragraph">
|
|
<p>Distributed tracing works by propagating fields inside and across services that
|
|
connect the trace together: traceId and spanId notably. The context that holds
|
|
these fields can optionally push other fields that need to be consistent
|
|
regardless of many services are touched. The simple name for these extra fields
|
|
is "Baggage".</p>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>Sleuth allows you to define which baggage are permitted to exist in the trace
|
|
context, including what header names are used.</p>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>The following example shows setting baggage values:</p>
|
|
</div>
|
|
<div class="listingblock">
|
|
<div class="content">
|
|
<pre class="highlightjs highlight"><code class="language-java hljs" data-lang="java">Span initialSpan = this.tracer.nextSpan().name("span").start();
|
|
BUSINESS_PROCESS.updateValue(initialSpan.context(), "ALM");
|
|
COUNTRY_CODE.updateValue(initialSpan.context(), "FO");</code></pre>
|
|
</div>
|
|
</div>
|
|
<div class="admonitionblock important">
|
|
<table>
|
|
<tr>
|
|
<td class="icon">
|
|
<i class="fa icon-important" title="Important"></i>
|
|
</td>
|
|
<td class="content">
|
|
There is currently no limitation of the count or size of baggage
|
|
items. Keep in mind that too many can decrease system throughput or increase
|
|
RPC latency. In extreme cases, too much baggage can crash the application, due
|
|
to exceeding transport-level message or header capacity.
|
|
</td>
|
|
</tr>
|
|
</table>
|
|
</div>
|
|
<div class="sect3">
|
|
<h4 id="baggage-versus-tags"><a class="anchor" href="#baggage-versus-tags"></a><a class="link" href="#baggage-versus-tags">1.4.1. Baggage versus Tags</a></h4>
|
|
<div class="paragraph">
|
|
<p>Like trace IDs, Baggage is attached to messages or requests, usually as
|
|
headers. Tags are key value pairs sent in a Span to Zipkin. Baggage values are
|
|
not added spans by default, which means you can’t search based on Baggage
|
|
unless you opt-in.</p>
|
|
</div>
|
|
<div class="paragraph">
|
|
<p>To make baggage also tags, use the property <code>spring.sleuth.baggage.tag-fields</code>
|
|
like so:</p>
|
|
</div>
|
|
<div class="listingblock">
|
|
<div class="content">
|
|
<pre class="highlightjs highlight"><code class="language-yml hljs" data-lang="yml">spring:
|
|
sleuth:
|
|
baggage:
|
|
remoteFields:
|
|
- country-code
|
|
- x-vcap-request-id
|
|
tagFields:
|
|
- country-code</code></pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<script type="text/javascript" src="js/tocbot/tocbot.min.js"></script>
|
|
<script type="text/javascript" src="js/toc.js"></script>
|
|
<link rel="stylesheet" href="js/highlight/styles/github.min.css">
|
|
<script src="js/highlight/highlight.min.js"></script>
|
|
<script>hljs.initHighlighting()</script>
|
|
</body>
|
|
</html> |