Added support for quartz instrumentation (#1428)

A running job should be wrapped in a tracing span. And if the JobDataMap for a given job is injected with tracing parameters, they will be extracted so the job will be a child span to that larger parent span.
This commit is contained in:
Branden Cash
2019-09-04 07:57:54 -07:00
committed by Marcin Grzejszczak
parent 3de0872992
commit 9a01b3974b
7 changed files with 695 additions and 3 deletions

View File

@@ -1229,7 +1229,7 @@ Spring Cloud Sleuth provides instrumentation for https://grpc.io/[gRPC] through
===== Dependencies
IMPORTANT: The gRPC integration relies on two external libraries to instrument clients and servers and both of those libraries must be on the class path to enable the instrumentation.
Maven:
Maven:
```
<dependency>
<groupId>io.github.lognet</groupId>
@@ -1248,7 +1248,7 @@ Gradle:
===== Server Instrumentation
Spring Cloud Sleuth leverages grpc-spring-boot-starter to register Brave's gRPC server interceptor with all services annotated with `@GRpcService`.
Spring Cloud Sleuth leverages grpc-spring-boot-starter to register Brave's gRPC server interceptor with all services annotated with `@GRpcService`.
===== Client Instrumentation
@@ -1388,6 +1388,12 @@ To disable Zuul support, set the `spring.sleuth.zuul.enabled` property to `false
We set `tracing` property to Lettcue `ClientResources` instance to enable Brave tracing built in Lettuce .
To disable Redis support, set the `spring.sleuth.redis.enabled` property to `false`.
=== Quartz
We instrument quartz jobs by adding Job/Trigger listeners to the Quartz Scheduler.
To turn off this feature, set the `spring.sleuth.quartz.enabled` property to `false`.
== Running examples
You can see the running examples deployed in the https://run.pivotal.io/[Pivotal Web Services].

View File

@@ -289,6 +289,13 @@
<optional>true</optional>
</dependency>
<!-- For Instrumentation of Quartz -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-quartz</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure-processor</artifactId>

View File

@@ -0,0 +1,64 @@
/*
* Copyright 2013-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.sleuth.instrument.quartz;
import brave.Tracing;
import org.quartz.Scheduler;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.quartz.QuartzAutoConfiguration;
import org.springframework.cloud.sleuth.autoconfig.TraceAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* {@link org.springframework.boot.autoconfigure.EnableAutoConfiguration
* Auto-configuration} enables Quartz span information propagation.
*
* @author Branden Cash
* @since 2.2.0
*/
@Configuration
@ConditionalOnBean({ Tracing.class, Scheduler.class })
@AutoConfigureAfter({ TraceAutoConfiguration.class, QuartzAutoConfiguration.class })
@ConditionalOnProperty(value = "spring.sleuth.quartz.enabled", matchIfMissing = true)
public class TraceQuartzAutoConfiguration implements InitializingBean {
private Scheduler scheduler;
private Tracing tracing;
public TraceQuartzAutoConfiguration(Scheduler scheduler, Tracing tracing) {
this.scheduler = scheduler;
this.tracing = tracing;
}
@Bean
public TracingJobListener tracingJobListener() {
return new TracingJobListener(tracing);
}
@Override
public void afterPropertiesSet() throws Exception {
scheduler.getListenerManager().addTriggerListener(tracingJobListener());
scheduler.getListenerManager().addJobListener(tracingJobListener());
}
}

View File

@@ -0,0 +1,120 @@
/*
* Copyright 2013-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.sleuth.instrument.quartz;
import brave.Span;
import brave.Tracer.SpanInScope;
import brave.Tracing;
import brave.propagation.Propagation.Getter;
import brave.propagation.TraceContextOrSamplingFlags;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.quartz.JobListener;
import org.quartz.Trigger;
import org.quartz.Trigger.CompletedExecutionInstruction;
import org.quartz.TriggerListener;
/**
* {@link org.quartz.JobListener JobListener} that will wrap a span around quartz jobs
* when they start and finish.
*
* @author Branden Cash
* @since 2.2.0
*/
class TracingJobListener implements JobListener, TriggerListener {
static final String TRIGGER_TAG_KEY = "quartz.trigger";
static final String CONTEXT_SPAN_KEY = Span.class.getName();
static final String CONTEXT_SPAN_IN_SCOPE_KEY = SpanInScope.class.getName();
private static final Getter<JobDataMap, String> GETTER = (carrier, key) -> {
Object value = carrier.get(key);
if (value instanceof String) {
return (String) value;
}
return null;
};
private final Tracing tracing;
TracingJobListener(Tracing tracing) {
this.tracing = tracing;
}
@Override
public String getName() {
return getClass().getName();
}
@Override
public void triggerFired(Trigger trigger, JobExecutionContext context) {
TraceContextOrSamplingFlags extracted = tracing.propagation().extractor(GETTER)
.extract(context.getMergedJobDataMap());
Span span = tracing.tracer().nextSpan(extracted)
.name(context.getTrigger().getJobKey().toString())
.tag(TRIGGER_TAG_KEY, context.getTrigger().getKey().toString());
context.put(CONTEXT_SPAN_KEY, span);
context.put(CONTEXT_SPAN_IN_SCOPE_KEY,
tracing.tracer().withSpanInScope(span.start()));
}
@Override
public boolean vetoJobExecution(Trigger trigger, JobExecutionContext context) {
return false;
}
@Override
public void triggerMisfired(Trigger trigger) {
}
@Override
public void triggerComplete(Trigger trigger, JobExecutionContext context,
CompletedExecutionInstruction triggerInstructionCode) {
closeTrace(context);
}
@Override
public void jobToBeExecuted(JobExecutionContext context) {
}
@Override
public void jobExecutionVetoed(JobExecutionContext context) {
closeTrace(context);
}
@Override
public void jobWasExecuted(JobExecutionContext context,
JobExecutionException jobException) {
}
private void closeTrace(JobExecutionContext context) {
Object spanInScope = context.get(CONTEXT_SPAN_IN_SCOPE_KEY);
Object span = context.get(CONTEXT_SPAN_KEY);
if (spanInScope instanceof SpanInScope) {
((SpanInScope) spanInScope).close();
}
if (span instanceof Span) {
((Span) span).finish();
}
}
}

View File

@@ -26,7 +26,8 @@ org.springframework.cloud.sleuth.instrument.messaging.TraceMessagingAutoConfigur
org.springframework.cloud.sleuth.instrument.messaging.TraceSpringIntegrationAutoConfiguration,\
org.springframework.cloud.sleuth.instrument.messaging.websocket.TraceWebSocketAutoConfiguration,\
org.springframework.cloud.sleuth.instrument.opentracing.OpentracingAutoConfiguration,\
org.springframework.cloud.sleuth.instrument.redis.TraceRedisAutoConfiguration
org.springframework.cloud.sleuth.instrument.redis.TraceRedisAutoConfiguration,\
org.springframework.cloud.sleuth.instrument.quartz.TraceQuartzAutoConfiguration
# Environment Post Processor
org.springframework.boot.env.EnvironmentPostProcessor=\
org.springframework.cloud.sleuth.autoconfig.TraceEnvironmentPostProcessor

View File

@@ -0,0 +1,139 @@
/*
* Copyright 2013-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.sleuth.instrument.quartz;
import brave.Tracer;
import brave.Tracing;
import org.junit.Test;
import org.quartz.ListenerManager;
import org.quartz.Scheduler;
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.autoconfigure.AutoConfigureBefore;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* @author Branden Cash
*/
public class TraceQuartzAutoConfigurationTest {
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(SchedulerConfig.class,
TracingConfig.class, TraceQuartzAutoConfiguration.class));
@Test
public void should_create_job_listener_bean_when_all_conditions_are_met() {
// when
this.contextRunner.run(context -> {
// expect
assertThat(context).hasSingleBean(TracingJobListener.class);
});
}
@Test
public void should_add_listener_to_trigger_listeners_when_conditions_are_met() {
// when
this.contextRunner.run(context -> {
// expect
verify(context.getBean(Scheduler.class).getListenerManager())
.addTriggerListener(context.getBean(TracingJobListener.class));
});
}
@Test
public void should_add_listener_job_listeners_when_conditions_are_met() {
// when
this.contextRunner.run(context -> {
// expect
verify(context.getBean(Scheduler.class).getListenerManager())
.addJobListener(context.getBean(TracingJobListener.class));
});
}
@Test
public void should_not_create_listener_bean_when_tracing_bean_is_not_present() {
new ApplicationContextRunner().withConfiguration(AutoConfigurations
// given
.of(SchedulerConfig.class, TraceQuartzAutoConfiguration.class))
// when
.run(context -> {
// expect
assertThat(context).doesNotHaveBean(TracingJobListener.class);
});
}
@Test
public void should_not_create_listener_when_scheduler_bean_is_not_present() {
new ApplicationContextRunner().withConfiguration(AutoConfigurations
// given
.of(TracingConfig.class, TraceQuartzAutoConfiguration.class))
// when
.run(context -> {
// expect
assertThat(context).doesNotHaveBean(TracingJobListener.class);
});
}
@Test
public void should_not_create_listener_when_sleuth_is_disabled() {
// when
this.contextRunner.withPropertyValues("spring.sleuth.quartz.enabled=false")
// expect
.run(context -> {
assertThat(context).doesNotHaveBean(TracingJobListener.class);
});
}
@Configuration
@AutoConfigureBefore(TraceQuartzAutoConfiguration.class)
public static class SchedulerConfig {
@Bean
public Scheduler scheduler() throws Exception {
Scheduler scheduler = mock(Scheduler.class);
when(scheduler.getListenerManager()).thenReturn(mock(ListenerManager.class));
return scheduler;
}
}
@Configuration
@AutoConfigureBefore(TraceQuartzAutoConfiguration.class)
public static class TracingConfig {
@Bean
public Tracing tracing() {
return mock(Tracing.class);
}
@Bean
public Tracer tracer() {
return mock(Tracer.class);
}
}
}

View File

@@ -0,0 +1,355 @@
/*
* Copyright 2013-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.sleuth.instrument.quartz;
import java.util.ArrayDeque;
import java.util.HashMap;
import java.util.Properties;
import java.util.Queue;
import java.util.concurrent.CompletableFuture;
import brave.Tracer.SpanInScope;
import brave.Tracing;
import brave.propagation.Propagation.Setter;
import brave.propagation.StrictScopeDecorator;
import brave.propagation.ThreadLocalCurrentTraceContext;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.quartz.Job;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.quartz.JobKey;
import org.quartz.JobListener;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.Trigger;
import org.quartz.Trigger.CompletedExecutionInstruction;
import org.quartz.TriggerKey;
import org.quartz.TriggerListener;
import org.quartz.impl.StdSchedulerFactory;
import org.quartz.listeners.JobListenerSupport;
import org.quartz.listeners.TriggerListenerSupport;
import org.quartz.utils.StringKeyDirtyFlagMap;
import zipkin2.Span;
import static org.assertj.core.api.Assertions.assertThat;
import static org.quartz.JobBuilder.newJob;
import static org.quartz.TriggerBuilder.newTrigger;
import static org.springframework.cloud.sleuth.instrument.quartz.TracingJobListener.CONTEXT_SPAN_IN_SCOPE_KEY;
import static org.springframework.cloud.sleuth.instrument.quartz.TracingJobListener.CONTEXT_SPAN_KEY;
import static org.springframework.cloud.sleuth.instrument.quartz.TracingJobListener.TRIGGER_TAG_KEY;
/**
* @author Branden Cash
*/
public class TracingJobListenerTest {
private static final JobKey SUCCESSFUL_JOB_KEY = new JobKey("SuccessfulJob");
private static final JobKey EXCEPTIONAL_JOB_KEY = new JobKey("ExceptionalJob");
private static final TriggerKey TRIGGER_KEY = new TriggerKey("ExampleTrigger");
private TracingJobListener listener;
private Tracing tracing;
private Scheduler scheduler;
private CompletableFuture completableJob;
private Queue<Span> spans = new ArrayDeque<>();
@Before
public void setUp() throws Exception {
tracing = Tracing.newBuilder().spanReporter(spans::add)
.currentTraceContext(ThreadLocalCurrentTraceContext.newBuilder()
.addScopeDecorator(StrictScopeDecorator.create()).build())
.build();
listener = new TracingJobListener(tracing);
completableJob = new CompleteableTriggerListener();
scheduler = createScheduler(getClass().getSimpleName(), 1);
scheduler.addJob(newJob(ExceptionalJob.class).withIdentity(EXCEPTIONAL_JOB_KEY)
.storeDurably().build(), true);
scheduler.addJob(newJob(SuccessfulJob.class).withIdentity(SUCCESSFUL_JOB_KEY)
.storeDurably().build(), true);
scheduler.getListenerManager().addTriggerListener(listener);
scheduler.getListenerManager().addJobListener(listener);
scheduler.getListenerManager()
.addTriggerListener((CompleteableTriggerListener) completableJob);
scheduler.getListenerManager()
.addJobListener((CompleteableTriggerListener) completableJob);
scheduler.start();
}
@After
public void tearDown() throws Exception {
scheduler.shutdown(true);
}
@Test
public void should_return_class_name_all_the_time() {
// when
String name = listener.getName();
// expect
assertThat(name).isEqualTo(TracingJobListener.class.getName());
}
@Test
public void should_complete_span_when_job_is_successful() throws Exception {
// given
Trigger trigger = newTrigger().forJob(SUCCESSFUL_JOB_KEY).startNow().build();
// when
runJob(trigger);
// expect
takeSpan();
}
@Test
public void should_have_span_with_proper_name_and_tag_when_job_is_successful()
throws Exception {
// given
Trigger trigger = newTrigger().withIdentity(TRIGGER_KEY)
.forJob(SUCCESSFUL_JOB_KEY).startNow().build();
// when
runJob(trigger);
// expect
Span span = takeSpan();
assertThat(span.name()).isEqualToIgnoringCase(SUCCESSFUL_JOB_KEY.toString());
assertThat(span.tags().get(TRIGGER_TAG_KEY))
.isEqualToIgnoringCase(TRIGGER_KEY.toString());
}
@Test
public void should_complete_span_when_job_throws_exception() throws Exception {
// given
Trigger trigger = newTrigger().forJob(EXCEPTIONAL_JOB_KEY).startNow().build();
// when
runJob(trigger);
// expect
takeSpan();
}
@Test
public void should_complete_span_when_job_is_vetoed() throws Exception {
// given
scheduler.getListenerManager().addTriggerListener(new VetoJobTriggerListener());
Trigger trigger = newTrigger().forJob(SUCCESSFUL_JOB_KEY).startNow().build();
// when
runJob(trigger);
// expect
takeSpan();
}
@Test
public void should_not_complete_span_when_context_is_modified_to_remove_keys()
throws Exception {
// given
scheduler.getListenerManager().addJobListener(new ContextModifyingJobListener());
Trigger trigger = newTrigger().forJob(EXCEPTIONAL_JOB_KEY).startNow().build();
// when
runJob(trigger);
// expect
requireNoSpan();
}
@Test
public void should_have_parent_and_child_span_when_trigger_contains_span_info()
throws Exception {
// given
brave.Span span = tracing.tracer().nextSpan();
JobDataMap data = new JobDataMap();
addSpanToJobData(data);
Trigger trigger = newTrigger().forJob(SUCCESSFUL_JOB_KEY).usingJobData(data)
.startNow().build();
// when
runJob(trigger);
// expect
Span parent = takeSpan();
Span child = takeSpan();
assertThat(parent.parentId()).isNull();
assertThat(child.parentId()).isEqualTo(parent.id());
}
@Test
public void should_have_parent_and_child_span_when_trigger_job_data_was_created_with_differently_typed_map()
throws Exception {
// given
JobDataMap data = new JobDataMap(new HashMap<Integer, Integer>());
addSpanToJobData(data);
Trigger trigger = newTrigger().forJob(SUCCESSFUL_JOB_KEY).usingJobData(data)
.startNow().build();
// when
runJob(trigger);
// expect
Span parent = takeSpan();
Span child = takeSpan();
assertThat(parent.parentId()).isNull();
assertThat(child.parentId()).isEqualTo(parent.id());
}
void runJob(Trigger trigger) throws SchedulerException {
scheduler.scheduleJob(trigger);
completableJob.join();
}
Scheduler createScheduler(String name, int threadPoolSize) throws SchedulerException {
Properties config = new Properties();
config.setProperty("org.quartz.scheduler.instanceName", name + "Scheduler");
config.setProperty("org.quartz.scheduler.instanceId", "AUTO");
config.setProperty("org.quartz.threadPool.threadCount",
Integer.toString(threadPoolSize));
config.setProperty("org.quartz.threadPool.class",
"org.quartz.simpl.SimpleThreadPool");
return new StdSchedulerFactory(config).getScheduler();
}
void addSpanToJobData(JobDataMap data) {
brave.Span span = tracing.tracer().nextSpan();
try (SpanInScope spanInScope = tracing.tracer().withSpanInScope(span)) {
tracing.propagation()
.injector((Setter<JobDataMap, String>) StringKeyDirtyFlagMap::put)
.inject(tracing.currentTraceContext().get(), data);
}
finally {
span.finish();
}
}
Span takeSpan() throws InterruptedException {
Span result = spans.poll();
assertThat(result).withFailMessage("Span was not reported, but was expected")
.isNotNull();
return result;
}
void requireNoSpan() throws InterruptedException {
Span result = spans.poll();
assertThat(result).withFailMessage("Span was reported, but was not expected")
.isNull();
}
public static class CompleteableTriggerListener extends CompletableFuture
implements TriggerListener, JobListener {
@Override
public String getName() {
return getClass().getName();
}
@Override
public void triggerFired(Trigger trigger, JobExecutionContext context) {
}
@Override
public boolean vetoJobExecution(Trigger trigger, JobExecutionContext context) {
return false;
}
@Override
public void triggerMisfired(Trigger trigger) {
}
@Override
public void triggerComplete(Trigger trigger, JobExecutionContext context,
CompletedExecutionInstruction triggerInstructionCode) {
complete(context.getResult());
}
@Override
public void jobToBeExecuted(JobExecutionContext context) {
}
@Override
public void jobExecutionVetoed(JobExecutionContext context) {
complete(context.getResult());
}
@Override
public void jobWasExecuted(JobExecutionContext context,
JobExecutionException jobException) {
}
}
public static class VetoJobTriggerListener extends TriggerListenerSupport {
@Override
public String getName() {
return getClass().getName();
}
@Override
public boolean vetoJobExecution(Trigger trigger, JobExecutionContext context) {
return true;
}
}
public static class ContextModifyingJobListener extends JobListenerSupport {
@Override
public String getName() {
return getClass().getName();
}
@Override
public void jobToBeExecuted(JobExecutionContext context) {
context.put(CONTEXT_SPAN_KEY, null);
context.put(CONTEXT_SPAN_IN_SCOPE_KEY, null);
}
}
public static class ExceptionalJob implements Job {
@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
throw new RuntimeException("Intentional Exception");
}
}
public static class SuccessfulJob implements Job {
@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
}
}
}