SpringCloud/spring-cloud-stream
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-2485: Improvements in Kafka Binder Metrics

Browse Source 
Resolves https://github.com/spring-cloud/spring-cloud-stream/issues/2485

These improvements are threading related

Initial implementation

Readd missing 'cache' for recent offset lags and general refactoring

Add a test case for the schedule-only variant and externalize scheduling interval

Consistency of configuration property naming and minor cleanup of existing code

Review feedback: documentation, author addition and copyright adjustments

Doc wording changes

Fix checkstyle issues

Adjust reference doc and improve new property names according to review

Move documentation to the correct file

Use JDK 8 compatible APIs and make the new test case public to satisfy junit
This commit is contained in:
Nico Heller
2022-08-29 21:18:43 +02:00
committed by Soby Chacko
parent 1964f31cfc
commit eaee63d660
4 changed files with 136 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
binders/kafka-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2015-2021 the original author or authors.
* Copyright 2015-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -62,6 +62,7 @@ import org.springframework.util.StringUtils;
* @author Aldo Sinanaj
* @author Lukasz Kaminski
* @author Chukwubuikem Ume-Ugwa
* @author Nico Heller
*/
@ConfigurationProperties(prefix = "spring.cloud.stream.kafka.binder")
public class KafkaBinderConfigurationProperties {
@@ -72,6 +73,8 @@ public class KafkaBinderConfigurationProperties {
private final Transaction transaction = new Transaction();
private final Metrics metrics = new Metrics();
private final KafkaProperties kafkaProperties;
/**
@@ -150,6 +153,10 @@ public class KafkaBinderConfigurationProperties {
return this.transaction;
}
public Metrics getMetrics() {
return metrics;
}
public String getKafkaConnectionString() {
// We need to do a check on certificate file locations to see if they are given as classpath resources.
// If that is the case, then we will move them to a file system location and use those as the certificate locations.
@@ -666,4 +673,34 @@ public class KafkaBinderConfigurationProperties {
}
public static class Metrics {
/**
* When set to true, the offset lag metric of each consumer topic is computed whenever the metric is queried (default: true).
* When set to false only the periodically calculated offset lag is used.
* See {@link #offsetLagMetricsInterval} for more information.
*/
private boolean defaultOffsetLagMetricsEnabled = true;
/**
* The interval in which the offset lag for each consumer topic is computed (default: 60 seconds).
* This value is used whenever {@link #defaultOffsetLagMetricsEnabled} is disabled or its computation is taking too long.
*/
private Duration offsetLagMetricsInterval = Duration.ofSeconds(60);
public boolean isDefaultOffsetLagMetricsEnabled() {
return defaultOffsetLagMetricsEnabled;
}
public void setDefaultOffsetLagMetricsEnabled(boolean defaultOffsetLagMetricsEnabled) {
this.defaultOffsetLagMetricsEnabled = defaultOffsetLagMetricsEnabled;
}
public Duration getOffsetLagMetricsInterval() {
return offsetLagMetricsInterval;
}
public void setOffsetLagMetricsInterval(Duration offsetLagMetricsInterval) {
this.offsetLagMetricsInterval = offsetLagMetricsInterval;
}
}
}

78
binders/kafka-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderMetrics.java
View File

@@ -30,6 +30,7 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.ToDoubleFunction;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
@@ -64,14 +65,13 @@ import org.springframework.util.ObjectUtils;
* @author Gary Russell
* @author Lars Bilger
* @author Tomek Szmytka
* @author Nico Heller
*/
public class KafkaBinderMetrics
implements MeterBinder, ApplicationListener<BindingCreatedEvent>, AutoCloseable {
private static final int DEFAULT_TIMEOUT = 5;
private static final int DELAY_BETWEEN_TASK_EXECUTION = 60;
private static final Log LOG = LogFactory.getLog(KafkaBinderMetrics.class);
/**
@@ -91,9 +91,9 @@ public class KafkaBinderMetrics
private int timeout = DEFAULT_TIMEOUT;
ScheduledExecutorService scheduler;
private final Map<String, Long> lastUnconsumedMessagesValues = new ConcurrentHashMap<>();
Map<String, Long> unconsumedMessages = new ConcurrentHashMap<>();
ScheduledExecutorService scheduler;
public KafkaBinderMetrics(KafkaMessageChannelBinder binder,
KafkaBinderConfigurationProperties binderConfigurationProperties,
@@ -141,58 +141,58 @@ public class KafkaBinderMetrics
String topic = topicInfo.getKey();
String group = topicInfo.getValue().getConsumerGroup();
final Gauge register = Gauge.builder(OFFSET_LAG_METRIC_NAME, this,
(o) -> computeAndGetUnconsumedMessages(topic, group)).tag("group", group)
.tag("topic", topic)
.description("Unconsumed messages for a particular group and topic")
.register(registry);
ToDoubleFunction<KafkaBinderMetrics> offsetComputation = computeOffsetComputationFunction(topic, group);
final Gauge register = Gauge.builder(OFFSET_LAG_METRIC_NAME, this, offsetComputation)
.tag("group", group)
.tag("topic", topic)
.description("Unconsumed messages for a particular group and topic")
.register(registry);
if (!(register instanceof NoopGauge)) {
//Schedule a task to compute the unconsumed messages for this group/topic every minute.
this.scheduler.scheduleWithFixedDelay(computeUnconsumedMessagesRunnable(topic, group, this.metadataConsumers),
10, DELAY_BETWEEN_TASK_EXECUTION, TimeUnit.SECONDS);
lastUnconsumedMessagesValues.put(topic + "-" + group, 0L);
this.scheduler.scheduleWithFixedDelay(
() -> computeUnconsumedMessages(topic, group),
1,
binderConfigurationProperties.getMetrics().getOffsetLagMetricsInterval().getSeconds(),
TimeUnit.SECONDS
);
}
}
}
private Runnable computeUnconsumedMessagesRunnable(String topic, String group, Map<String, Consumer<?, ?>> metadataConsumers) {
return () -> {
try {
long lag = findTotalTopicGroupLag(topic, group, this.metadataConsumers);
this.unconsumedMessages.put(topic + "-" + group, lag);
}
catch (Exception ex) {
LOG.debug("Cannot generate metric for topic: " + topic, ex);
}
};
private ToDoubleFunction<KafkaBinderMetrics> computeOffsetComputationFunction(String topic, String group) {
if (this.binderConfigurationProperties.getMetrics().isDefaultOffsetLagMetricsEnabled()) {
return (o) -> computeAndGetUnconsumedMessagesWithTimeout(topic, group);
}
else {
return (o) -> lastUnconsumedMessagesValues.get(topic + "-" + group);
}
}
private long computeAndGetUnconsumedMessages(String topic, String group) {
ExecutorService exec = Executors.newCachedThreadPool();
Future<Long> future = exec.submit(() -> {
long lag = 0;
try {
lag = findTotalTopicGroupLag(topic, group, this.metadataConsumers);
}
catch (Exception ex) {
LOG.debug("Cannot generate metric for topic: " + topic, ex);
}
return lag;
});
private long computeAndGetUnconsumedMessagesWithTimeout(String topic, String group) {
Future<Long> future = scheduler.submit(() -> computeUnconsumedMessages(topic, group));
try {
return future.get(this.timeout, TimeUnit.SECONDS);
}
catch (InterruptedException ex) {
Thread.currentThread().interrupt();
return this.unconsumedMessages.getOrDefault(topic + "-" + group, 0L);
return lastUnconsumedMessagesValues.get(topic + "-" + group);
}
catch (ExecutionException | TimeoutException ex) {
return this.unconsumedMessages.getOrDefault(topic + "-" + group, 0L);
return lastUnconsumedMessagesValues.get(topic + "-" + group);
}
finally {
exec.shutdownNow();
}
private long computeUnconsumedMessages(String topic, String group) {
long lag = 0;
try {
lag = findTotalTopicGroupLag(topic, group, this.metadataConsumers);
this.lastUnconsumedMessagesValues.put(topic + "-" + group, lag);
}
catch (Exception ex) {
LOG.debug("Cannot generate metric for topic: " + topic, ex);
}
return lag;
}
private long findTotalTopicGroupLag(String topic, String group, Map<String, Consumer<?, ?>> metadataConsumers) {

43
binders/kafka-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderMetricsTest.java
View File

@@ -16,6 +16,7 @@
package org.springframework.cloud.stream.binder.kafka;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -32,8 +33,10 @@ import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
import org.awaitility.Awaitility;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Answers;
import org.mockito.ArgumentMatchers;
import org.mockito.Mock;
import org.mockito.Mockito;
@@ -52,6 +55,7 @@ import static org.mockito.Mockito.mock;
* @author Soby Chacko
* @author Lars Bilger
* @author Tomek Szmytka
* @author Nico Heller
*/
public class KafkaBinderMetricsTest {
@@ -72,7 +76,7 @@ public class KafkaBinderMetricsTest {
private Map<String, TopicInformation> topicsInUse = new HashMap<>();
@Mock
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties;
@Before
@@ -82,6 +86,10 @@ public class KafkaBinderMetricsTest {
.createConsumer(ArgumentMatchers.any(), ArgumentMatchers.any()))
.willReturn(consumer);
org.mockito.BDDMockito.given(binder.getTopicsInUse()).willReturn(topicsInUse);
org.mockito.BDDMockito.given(kafkaBinderConfigurationProperties.getMetrics().isDefaultOffsetLagMetricsEnabled())
.willReturn(true);
org.mockito.BDDMockito.given(kafkaBinderConfigurationProperties.getMetrics().getOffsetLagMetricsInterval())
.willReturn(Duration.ofSeconds(60));
metrics = new KafkaBinderMetrics(binder, kafkaBinderConfigurationProperties,
consumerFactory, null
);
@@ -113,6 +121,39 @@ public class KafkaBinderMetricsTest {
.isEqualTo(500.0);
}
@Test
public void shouldFallbackToScheduledOffsetLagComputationWhenRealtimeOffsetLagIsDisabled() {
final Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>();
TopicPartition topicPartition = new TopicPartition(TEST_TOPIC, 0);
committed.put(topicPartition, new OffsetAndMetadata(500));
org.mockito.BDDMockito
.given(consumer.committed(ArgumentMatchers.anySet()))
.willReturn(committed);
List<PartitionInfo> partitions = partitions(new Node(0, null, 0));
topicsInUse.put(
TEST_TOPIC,
new TopicInformation("group1-metrics", partitions, false)
);
org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC))
.willReturn(partitions);
org.mockito.BDDMockito.given(kafkaBinderConfigurationProperties.getMetrics().isDefaultOffsetLagMetricsEnabled())
.willReturn(false);
metrics.bindTo(meterRegistry);
assertThat(meterRegistry.getMeters()).hasSize(1);
assertThat(meterRegistry.get(KafkaBinderMetrics.OFFSET_LAG_METRIC_NAME)
.tag("group", "group1-metrics").tag("topic", TEST_TOPIC).gauge().value())
.isEqualTo(0);
org.mockito.BDDMockito.given(kafkaBinderConfigurationProperties.getMetrics().getOffsetLagMetricsInterval())
.willReturn(Duration.ofSeconds(1));
metrics.bindTo(meterRegistry);
Awaitility.waitAtMost(Duration.ofSeconds(5)).untilAsserted(() -> {
assertThat(meterRegistry.get(KafkaBinderMetrics.OFFSET_LAG_METRIC_NAME)
.tag("group", "group1-metrics").tag("topic", TEST_TOPIC).gauge().value())
.isEqualTo(500);
});
}
@Test
public void shouldNotContainAnyMetricsWhenUsingNoopGauge() {
// Adding NoopGauge for the offset metric.