Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use spring-kafka in s-i-kafka

Browse Source 
Update to s-i 1.0.0.M1

Polishing

Fix Readme
This commit is contained in:
Gary Russell
2016-02-25 16:24:23 -05:00
committed by Artem Bilan
parent 49e533410c
commit a030be3d7e
154 changed files with 205 additions and 16781 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParser.java
View File

@@ -1,156 +0,0 @@
/*
* Copyright 2002-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.w3c.dom.Element;
import org.springframework.beans.BeanMetadataElement;
import org.springframework.beans.factory.config.BeanDefinition;
import org.springframework.beans.factory.support.AbstractBeanDefinition;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.support.ManagedMap;
import org.springframework.beans.factory.xml.AbstractSingleBeanDefinitionParser;
import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.util.StringUtils;
import org.springframework.util.xml.DomUtils;
/**
* @author Soby Chacko
* @author Rajasekar Elango
* @author Artem Bilan
* @author Ilayaperumal Gopinathan
* @author Gary Russell
* @since 0.5
*/
@Deprecated
@SuppressWarnings("deprecation")
public class KafkaConsumerContextParser extends AbstractSingleBeanDefinitionParser {
@Override
protected Class<?> getBeanClass(final Element element) {
return org.springframework.integration.kafka.support.KafkaConsumerContext.class;
}
@Override
protected void doParse(final Element element, final ParserContext parserContext, final BeanDefinitionBuilder builder) {
super.doParse(element, parserContext, builder);
final Element consumerConfigurations = DomUtils.getChildElementByTagName(element, "consumer-configurations");
parseConsumerConfigurations(consumerConfigurations, parserContext, builder, element);
}
private void parseConsumerConfigurations(final Element consumerConfigurations, final ParserContext parserContext,
final BeanDefinitionBuilder builder, final Element parentElem) {
Map<String, BeanMetadataElement> consumerConfigurationsMap = new ManagedMap<String, BeanMetadataElement>();
for (final Element consumerConfiguration : DomUtils.getChildElementsByTagName(consumerConfigurations, "consumer-configuration")) {
final BeanDefinitionBuilder consumerConfigurationBuilder =
BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.support.ConsumerConfiguration.class);
final BeanDefinitionBuilder consumerMetadataBuilder =
BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.support.ConsumerMetadata.class);
IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration,
"group-id");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration,
"value-decoder");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration,
"key-decoder");
IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerConfigurationBuilder, consumerConfiguration,
"max-messages");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(consumerConfigurationBuilder, consumerConfiguration,
"executor");
IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerMetadataBuilder, parentElem,
"consumer-timeout");
final Map<String, String> topicStreamsMap = new HashMap<String, String>();
final List<Element> topicConfigurations = DomUtils.getChildElementsByTagName(consumerConfiguration, "topic");
if (topicConfigurations != null) {
for (final Element topicConfiguration : topicConfigurations) {
final String topic = topicConfiguration.getAttribute("id");
final String streams = topicConfiguration.getAttribute("streams");
topicStreamsMap.put(topic, streams);
}
consumerMetadataBuilder.addPropertyValue("topicStreamMap", topicStreamsMap);
}
final Element topicFilter = DomUtils.getChildElementByTagName(consumerConfiguration, "topic-filter");
if (topicFilter != null) {
BeanDefinition topicFilterConfigurationBeanDefinition =
BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.support.TopicFilterConfiguration.class)
.addConstructorArgValue(topicFilter.getAttribute("pattern"))
.addConstructorArgValue(topicFilter.getAttribute("streams"))
.addConstructorArgValue(topicFilter.getAttribute("exclude"))
.getBeanDefinition();
consumerMetadataBuilder.addPropertyValue("topicFilterConfiguration",
topicFilterConfigurationBeanDefinition);
}
final AbstractBeanDefinition consumerMetadataBeanDefintiion = consumerMetadataBuilder.getBeanDefinition();
final String zookeeperConnectBean = parentElem.getAttribute("zookeeper-connect");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, parentElem, zookeeperConnectBean);
final String consumerPropertiesBean = parentElem.getAttribute("consumer-properties");
final BeanDefinitionBuilder consumerConfigFactoryBuilder =
BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.support.ConsumerConfigFactoryBean.class);
consumerConfigFactoryBuilder.addConstructorArgValue(consumerMetadataBeanDefintiion);
if (StringUtils.hasText(zookeeperConnectBean)) {
consumerConfigFactoryBuilder.addConstructorArgReference(zookeeperConnectBean);
}
if (StringUtils.hasText(consumerPropertiesBean)) {
consumerConfigFactoryBuilder.addConstructorArgReference(consumerPropertiesBean);
}
AbstractBeanDefinition consumerConfigFactoryBuilderBeanDefinition =
consumerConfigFactoryBuilder.getBeanDefinition();
BeanDefinitionBuilder consumerConnectionProviderBuilder =
BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.support.ConsumerConnectionProvider.class);
consumerConnectionProviderBuilder.addConstructorArgValue(consumerConfigFactoryBuilderBeanDefinition);
AbstractBeanDefinition consumerConnectionProviderBuilderBeanDefinition =
consumerConnectionProviderBuilder.getBeanDefinition();
BeanDefinitionBuilder messageLeftOverBeanDefinitionBuilder =
BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.support.MessageLeftOverTracker.class);
AbstractBeanDefinition messageLeftOverBeanDefinition =
messageLeftOverBeanDefinitionBuilder.getBeanDefinition();
consumerConfigurationBuilder.addConstructorArgValue(consumerMetadataBeanDefintiion);
consumerConfigurationBuilder.addConstructorArgValue(consumerConnectionProviderBuilderBeanDefinition);
consumerConfigurationBuilder.addConstructorArgValue(messageLeftOverBeanDefinition);
AbstractBeanDefinition consumerConfigurationBeanDefinition =
consumerConfigurationBuilder.getBeanDefinition();
consumerConfigurationsMap.put(consumerConfiguration.getAttribute("group-id"),
consumerConfigurationBeanDefinition);
}
builder.addPropertyValue("consumerConfigurations", consumerConfigurationsMap);
}
}

53
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaInboundChannelAdapterParser.java
View File

@@ -1,53 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import org.w3c.dom.Element;
import org.springframework.beans.BeanMetadataElement;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.util.StringUtils;
/**
* The Kafka Inbound Channel adapter parser
*
* @author Soby Chacko
* @since 0.5
*
*/
@Deprecated
public class KafkaInboundChannelAdapterParser extends AbstractPollingInboundChannelAdapterParser {
@Override
@SuppressWarnings("deprecation")
protected BeanMetadataElement parseSource(final Element element, final ParserContext parserContext) {
final BeanDefinitionBuilder highLevelConsumerMessageSourceBuilder =
BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.inbound.KafkaHighLevelConsumerMessageSource.class);
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(highLevelConsumerMessageSourceBuilder, element, "kafka-decoder");
final String kafkaConsumerContext = element.getAttribute("kafka-consumer-context-ref");
if (StringUtils.hasText(kafkaConsumerContext)) {
highLevelConsumerMessageSourceBuilder.addConstructorArgReference(kafkaConsumerContext);
}
return highLevelConsumerMessageSourceBuilder.getBeanDefinition();
}
}

57
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParser.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2015 the original author or authors
* Copyright 2015-2016 the original author or authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -24,11 +24,11 @@ import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.xml.AbstractChannelAdapterParser;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter;
import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer;
import org.springframework.util.StringUtils;
/**
* @author Artem Bilan.
* @author Artem Bilan
* @author Gary Russell
*/
public class KafkaMessageDrivenChannelAdapterParser extends AbstractChannelAdapterParser {
@@ -38,65 +38,16 @@ public class KafkaMessageDrivenChannelAdapterParser extends AbstractChannelAdapt
BeanDefinitionBuilder.genericBeanDefinition(KafkaMessageDrivenChannelAdapter.class);
String container = element.getAttribute("listener-container");
String connectionFactory = element.getAttribute("connection-factory");
String topics = element.getAttribute("topics");
String offsetManager = element.getAttribute("offset-manager");
String errorHandler = element.getAttribute("error-handler");
String taskExecutor = element.getAttribute("task-executor");
String concurrency = element.getAttribute("concurrency");
String stopTimeout = element.getAttribute("stop-timeout");
String maxFetch = element.getAttribute("max-fetch");
String queueSize = element.getAttribute("queue-size");
if (StringUtils.hasText(container) &&
(StringUtils.hasText(connectionFactory) || StringUtils.hasText(topics)
|| StringUtils.hasText(offsetManager) || StringUtils.hasText(errorHandler)
|| StringUtils.hasText(taskExecutor) || StringUtils.hasText(concurrency)
|| StringUtils.hasText(maxFetch) || StringUtils.hasText(queueSize)
|| StringUtils.hasText(stopTimeout))) {
parserContext.getReaderContext().error("The 'listener-container' is mutually exclusive with " +
"'connection-factory', 'topics', 'offset-manager', 'error-handler', 'task-executor', " +
"'concurrency', 'stop-timeout', 'max-fetch' and 'queue-size'.", element);
}
if (StringUtils.hasText(container)) {
builder.addConstructorArgReference(container);
}
else {
if (!StringUtils.hasText(connectionFactory)) {
parserContext.getReaderContext().error("The 'connection-factory' attribute is required when " +
"'listener-container' isn't provided.", element);
}
if (!StringUtils.hasText(topics)) {
parserContext.getReaderContext().error("The 'topics' attribute is required when " +
"'listener-container' isn't provided.", element);
}
BeanDefinitionBuilder containerBuilder =
BeanDefinitionBuilder.genericBeanDefinition(KafkaMessageListenerContainer.class);
containerBuilder.addConstructorArgReference(connectionFactory)
.addConstructorArgValue(topics);
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(containerBuilder, element, "offset-manager");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(containerBuilder, element, "error-handler");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(
containerBuilder, element, "task-executor", "fetchTaskExecutor");
IntegrationNamespaceUtils.setValueIfAttributeDefined(containerBuilder, element, "concurrency");
IntegrationNamespaceUtils.setValueIfAttributeDefined(containerBuilder, element, "max-fetch");
IntegrationNamespaceUtils.setValueIfAttributeDefined(containerBuilder, element, "stop-timeout");
IntegrationNamespaceUtils.setValueIfAttributeDefined(containerBuilder, element, "queue-size");
IntegrationNamespaceUtils.setValueIfAttributeDefined(containerBuilder, element, "auto-commit-on-error");
builder.addConstructorArgValue(containerBuilder.getBeanDefinition());
parserContext.getReaderContext().error("The 'listener-container' attribute is required.", element);
}
builder.addPropertyReference("outputChannel", channelName);
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "send-timeout");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, element, "error-channel");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, element, "key-decoder");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, element, "payload-decoder");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element,
"auto-commit", "autoCommitOffset");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element,
"use-context-message-builder", "useMessageBuilderFactory");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element,

8
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaNamespaceHandler.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2015 the original author or authors.
* Copyright 2002-2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,19 +22,15 @@ import org.springframework.integration.config.xml.AbstractIntegrationNamespaceHa
* The namespace handler for the Kafka namespace
*
* @author Soby Chacko
* @author Gary Russell
* @since 0.5
*
*/
public class KafkaNamespaceHandler extends AbstractIntegrationNamespaceHandler {
@Override
@SuppressWarnings("deprecation")
public void init() {
registerBeanDefinitionParser("zookeeper-connect", new ZookeeperConnectParser());
registerBeanDefinitionParser("inbound-channel-adapter", new KafkaInboundChannelAdapterParser());
registerBeanDefinitionParser("outbound-channel-adapter", new KafkaOutboundChannelAdapterParser());
registerBeanDefinitionParser("producer-context", new KafkaProducerContextParser());
registerBeanDefinitionParser("consumer-context", new KafkaConsumerContextParser());
registerBeanDefinitionParser("message-driven-channel-adapter", new KafkaMessageDrivenChannelAdapterParser());
}

11
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaOutboundChannelAdapterParser.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2013-2014 the original author or authors.
* Copyright 2013-2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import org.w3c.dom.Element;
@@ -24,12 +25,12 @@ import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.xml.AbstractOutboundChannelAdapterParser;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler;
import org.springframework.util.StringUtils;
/**
*
* @author Soby Chacko
* @author Artem Bilan
* @author Gary Russell
* @since 0.5
*
*/
@@ -40,11 +41,9 @@ public class KafkaOutboundChannelAdapterParser extends AbstractOutboundChannelAd
final BeanDefinitionBuilder kafkaProducerMessageHandlerBuilder =
BeanDefinitionBuilder.genericBeanDefinition(KafkaProducerMessageHandler.class);
final String kafkaServerBeanName = element.getAttribute("kafka-producer-context-ref");
final String kafkaTemplateBeanName = element.getAttribute("kafka-template");
if (StringUtils.hasText(kafkaServerBeanName)) {
kafkaProducerMessageHandlerBuilder.addConstructorArgReference(kafkaServerBeanName);
}
kafkaProducerMessageHandlerBuilder.addConstructorArgReference(kafkaTemplateBeanName);
BeanDefinition topicExpressionDef =
IntegrationNamespaceUtils.createExpressionDefinitionFromValueOrExpression("topic", "topic-expression",

172
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParser.java
View File

@@ -1,172 +0,0 @@
/*
* Copyright 2002-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.w3c.dom.Element;
import org.springframework.beans.BeanMetadataElement;
import org.springframework.beans.factory.support.AbstractBeanDefinition;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.support.ManagedMap;
import org.springframework.beans.factory.xml.AbstractSimpleBeanDefinitionParser;
import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.integration.kafka.support.KafkaProducerContext;
import org.springframework.integration.kafka.support.ProducerConfiguration;
import org.springframework.integration.kafka.support.ProducerFactoryBean;
import org.springframework.integration.kafka.support.ProducerMetadata;
import org.springframework.integration.kafka.util.EncoderAdaptingSerializer;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
import org.springframework.util.xml.DomUtils;
/**
* @author Soby Chacko
* @author Ilayaperumal Gopinathan
* @author Gary Russell
* @author Artem Bilan
* @since 0.5
*/
public class KafkaProducerContextParser extends AbstractSimpleBeanDefinitionParser {
private static final Log log = LogFactory.getLog(KafkaProducerContextParser.class);
@Override
protected Class<?> getBeanClass(final Element element) {
return KafkaProducerContext.class;
}
@Override
protected void doParse(final Element element, final ParserContext parserContext, final BeanDefinitionBuilder builder) {
super.doParse(element, parserContext, builder);
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "phase");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "auto-startup");
final Element topics = DomUtils.getChildElementByTagName(element, "producer-configurations");
parseProducerConfigurations(topics, parserContext, builder, element);
}
@Override
protected boolean isEligibleAttribute(String attributeName) {
return !"producer-properties".equals(attributeName) && super.isEligibleAttribute(attributeName);
}
private void parseProducerConfigurations(Element topics, ParserContext parserContext,
BeanDefinitionBuilder builder, Element parentElem) {
Map<String, BeanMetadataElement> producerConfigurationsMap = new ManagedMap<String, BeanMetadataElement>();
for (Element producerConfiguration : DomUtils.getChildElementsByTagName(topics, "producer-configuration")) {
BeanDefinitionBuilder producerMetadataBuilder =
BeanDefinitionBuilder.genericBeanDefinition(ProducerMetadata.class);
producerMetadataBuilder.addConstructorArgValue(producerConfiguration.getAttribute("topic"));
producerMetadataBuilder.addConstructorArgValue(producerConfiguration.getAttribute("key-class-type"));
producerMetadataBuilder.addConstructorArgValue(producerConfiguration.getAttribute("value-class-type"));
String keySerializer = producerConfiguration.getAttribute("key-serializer");
String keyEncoder = producerConfiguration.getAttribute("key-encoder");
Assert.isTrue((StringUtils.hasText(keySerializer) ^ StringUtils.hasText(keyEncoder)),
"Exactly one of 'key-serializer' or 'key-encoder' must be specified");
if (StringUtils.hasText(keyEncoder)) {
if (log.isWarnEnabled()) {
log.warn("'key-encoder' is a deprecated option, use 'key-serializer' instead.");
}
BeanDefinitionBuilder encoderAdaptingSerializerBean = BeanDefinitionBuilder.genericBeanDefinition(EncoderAdaptingSerializer.class);
encoderAdaptingSerializerBean.addConstructorArgReference(keyEncoder);
producerMetadataBuilder.addConstructorArgValue(encoderAdaptingSerializerBean.getBeanDefinition());
}
else {
producerMetadataBuilder.addConstructorArgReference(keySerializer);
}
String valueSerializer = producerConfiguration.getAttribute("value-serializer");
String valueEncoder = producerConfiguration.getAttribute("value-encoder");
Assert.isTrue((StringUtils.hasText(valueSerializer) ^ StringUtils.hasText(valueEncoder)),
"Exactly one of 'value-serializer' or 'value-encoder' must be specified");
if (StringUtils.hasText(valueEncoder)) {
if (log.isWarnEnabled()) {
log.warn("'value-encoder' is a deprecated option, use 'value-serializer' instead.");
}
BeanDefinitionBuilder encoderAdaptingSerializerBean =
BeanDefinitionBuilder.genericBeanDefinition(EncoderAdaptingSerializer.class);
encoderAdaptingSerializerBean.addConstructorArgReference(valueEncoder);
producerMetadataBuilder.addConstructorArgValue(encoderAdaptingSerializerBean.getBeanDefinition());
}
else {
producerMetadataBuilder.addConstructorArgReference(valueSerializer);
}
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(producerMetadataBuilder, producerConfiguration,
"partitioner");
if (StringUtils.hasText(producerConfiguration.getAttribute("partitioner"))) {
if (log.isWarnEnabled()) {
log.warn("'partitioner' is a deprecated option. Use the 'kafka_partitionId' message header or " +
"the 'partition-id' (or 'partition-id-expression') attribute.");
}
}
IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration,
"compression-type");
IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration,
"batch-bytes");
IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration,
"sync");
IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration,
"send-timeout");
IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration,
"charset");
AbstractBeanDefinition producerMetadataBeanDefinition = producerMetadataBuilder.getBeanDefinition();
String producerPropertiesBean = parentElem.getAttribute("producer-properties");
BeanDefinitionBuilder producerFactoryBuilder =
BeanDefinitionBuilder.genericBeanDefinition(ProducerFactoryBean.class);
producerFactoryBuilder.addConstructorArgValue(producerMetadataBeanDefinition);
final String brokerList = producerConfiguration.getAttribute("broker-list");
if (StringUtils.hasText(brokerList)) {
producerFactoryBuilder.addConstructorArgValue(producerConfiguration.getAttribute("broker-list"));
}
if (StringUtils.hasText(producerPropertiesBean)) {
producerFactoryBuilder.addConstructorArgReference(producerPropertiesBean);
}
AbstractBeanDefinition producerFactoryBeanDefinition = producerFactoryBuilder.getBeanDefinition();
BeanDefinitionBuilder producerConfigurationBuilder =
BeanDefinitionBuilder.genericBeanDefinition(ProducerConfiguration.class)
.addConstructorArgValue(producerMetadataBeanDefinition)
.addConstructorArgValue(producerFactoryBeanDefinition);
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(producerConfigurationBuilder, producerConfiguration,
"conversion-service");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(producerConfigurationBuilder, producerConfiguration,
"producer-listener");
producerConfigurationsMap.put(producerConfiguration.getAttribute("topic"),
producerConfigurationBuilder.getBeanDefinition());
}
builder.addPropertyValue("producerConfigurations", producerConfigurationsMap);
}
}

47
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParser.java
View File

@@ -1,47 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.xml.AbstractSimpleBeanDefinitionParser;
import org.springframework.beans.factory.xml.BeanDefinitionParserDelegate;
import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.integration.kafka.support.ZookeeperConnect;
import org.w3c.dom.Element;
/**
* @author Soby Chacko
* @since 0.5
*/
public class ZookeeperConnectParser extends AbstractSimpleBeanDefinitionParser {
@Override
protected Class<?> getBeanClass(final Element element) {
return ZookeeperConnect.class;
}
@Override
protected void doParse(final Element element, final ParserContext parserContext, final BeanDefinitionBuilder builder) {
super.doParse(element, parserContext, builder);
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element,
BeanDefinitionParserDelegate.SCOPE_ATTRIBUTE);
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-connect");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-connection-timeout");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-session-timeout");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-sync-time");
}
}

187
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/AbstractConfiguration.java
View File

@@ -1,187 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import java.util.List;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
/**
* Default implementation of a {@link Configuration}, storing the default topic and partitions,
* as well as connectivity parameters.
*
* Implementors must provide a strategy for retrieving the seed brokers.
*
* @author Marius Bogoevici
*/
public abstract class AbstractConfiguration implements InitializingBean, Configuration {
private List<Partition> defaultPartitions;
private String defaultTopic;
private String clientId = KafkaConsumerDefaults.GROUP_ID;
private int minBytes = KafkaConsumerDefaults.MIN_FETCH_BYTES;
private int maxWait = KafkaConsumerDefaults.MAX_WAIT_TIME_IN_MS;
private int bufferSize = KafkaConsumerDefaults.SOCKET_BUFFER_SIZE_INT;
private int socketTimeout = KafkaConsumerDefaults.SOCKET_TIMEOUT_INT;
private int backoff = KafkaConsumerDefaults.BACKOFF_INCREMENT_INT;
private int fetchMetadataTimeout = KafkaConsumerDefaults.FETCH_METADATA_TIMEOUT;
/**
* The minimum amount of data that a server fetch operation will wait for before returning,
* unless {@code maxWait} has elapsed.
* In conjunction with {@link Configuration#getMaxWait()}}, controls latency
* and throughput.
* Smaller values increase responsiveness, but may increase the number of poll operations,
* potentially reducing throughput and increasing CPU consumption.
* @param minBytes the amount of data to fetch
*/
public void setMinBytes(int minBytes) {
this.minBytes = minBytes;
}
@Override
public int getMinBytes() {
return this.minBytes;
}
/**
* The maximum amount of time that a server fetch operation will wait before returning
* (unless {@code minFetchSizeInBytes}) are available.
* In conjunction with {@link AbstractConfiguration#setMinBytes(int)},
* controls latency and throughput.
* Smaller intervals increase responsiveness, but may increase
* the number of poll operations, potentially increasing CPU
* consumption and reducing throughput.
* @param maxWait timeout to wait
*/
public void setMaxWait(int maxWait) {
this.maxWait = maxWait;
}
@Override
public int getMaxWait() {
return this.maxWait;
}
@Override
public String getClientId() {
return this.clientId;
}
/**
* A client name to be used throughout this connection.
* @param clientId the client name
*/
public void setClientId(String clientId) {
this.clientId = clientId;
}
@Override
public int getBufferSize() {
return this.bufferSize;
}
/**
* The buffer size for this client
* @param bufferSize the buffer size
*/
public void setBufferSize(int bufferSize) {
this.bufferSize = bufferSize;
}
@Override
public int getSocketTimeout() {
return this.socketTimeout;
}
/**
* The socket timeout for this client
* @param socketTimeout the socket timeout
*/
public void setSocketTimeout(int socketTimeout) {
this.socketTimeout = socketTimeout;
}
@Override
public int getBackOff() {
return backoff;
}
/**
* The retry backoff time for this client
* @param backoff the retry backoff
*/
public void setBackoff(int backoff) {
this.backoff = backoff;
}
/**
* The timeout on fetching metadata (e.g. partition leaders)
* @param fetchMetadataTimeout timeout
*/
public void setFetchMetadataTimeout(int fetchMetadataTimeout) {
this.fetchMetadataTimeout = fetchMetadataTimeout;
}
@Override
public int getFetchMetadataTimeout() {
return this.fetchMetadataTimeout;
}
@Override
public final List<BrokerAddress> getBrokerAddresses() {
return doGetBrokerAddresses();
}
@Override
public List<Partition> getDefaultPartitions() {
return this.defaultPartitions;
}
public void setDefaultPartitions(List<Partition> defaultPartitions) {
this.defaultPartitions = defaultPartitions;
}
@Override
public String getDefaultTopic() {
return this.defaultTopic;
}
public void setDefaultTopic(String defaultTopic) {
this.defaultTopic = defaultTopic;
}
@Override
public void afterPropertiesSet() throws Exception {
Assert.isTrue(CollectionUtils.isEmpty(this.defaultPartitions) || StringUtils.isEmpty(this.defaultTopic)
, "A list of default partitions or a default topic may be specified, but not both");
}
protected abstract List<BrokerAddress> doGetBrokerAddresses();
}

100
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddress.java
View File

@@ -1,100 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import org.springframework.util.Assert;
import kafka.cluster.Broker;
/**
* Encapsulates the address of a Kafka broker.
*
* @author Marius Bogoevici
*/
public class BrokerAddress {
public static final int DEFAULT_PORT = 9092;
private final String host;
private final int port;
public BrokerAddress(String host, int port) {
Assert.hasText(host, "Host cannot be empty");
this.host = host;
this.port = port;
}
public BrokerAddress(String host) {
this(host, DEFAULT_PORT);
}
public BrokerAddress(Broker broker) {
Assert.notNull(broker, "Broker cannot be null");
this.host = broker.host();
this.port = broker.port();
}
public static BrokerAddress fromAddress(String address) {
String[] split = address.split(":");
if (split.length == 0 || split.length > 2) {
throw new IllegalArgumentException("Expected format <host>[:<port>]");
}
if (split.length == 2) {
return new BrokerAddress(split[0], Integer.parseInt(split[1]));
}
else {
return new BrokerAddress(split[0]);
}
}
public String getHost() {
return this.host;
}
public int getPort() {
return this.port;
}
@Override
public int hashCode() {
return 31 * this.host.hashCode() + this.port;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
BrokerAddress brokerAddress = (BrokerAddress) o;
return this.port == brokerAddress.port && this.host.equals(brokerAddress.host);
}
@Override
public String toString() {
return this.host + ":" + this.port;
}
}

41
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddressListConfiguration.java
View File

@@ -1,41 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import java.util.Arrays;
import java.util.List;
/**
* Kafka {@link Configuration} where the seed brokers are set up explicitly.
*
* @author Marius Bogoevici
*/
public class BrokerAddressListConfiguration extends AbstractConfiguration {
private final List<BrokerAddress> brokerAddresses;
public BrokerAddressListConfiguration(BrokerAddress... brokerAddresses) {
this.brokerAddresses = Arrays.asList(brokerAddresses);
}
@Override
protected List<BrokerAddress> doGetBrokerAddresses() {
return brokerAddresses;
}
}

101
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Configuration.java
View File

@@ -1,101 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import java.util.List;
/**
* Used to configure a {@link DefaultConnectionFactory}. Provides a list of seed brokers.
*
* @author Marius Bogoevici
*/
public interface Configuration {
/**
* The minimum amount of data that a server fetch operation will wait for before returning,
* unless {@code maxWait} has elapsed.
* In conjunction with {@link Configuration#getMaxWait()}}, controls latency
* and throughput.
* Smaller values increase responsiveness, but may increase the number of poll operations,
* potentially reducing throughput and increasing CPU consumption.
* @return the minimum amount of data for a fetch operation
*/
int getMinBytes();
/**
* The maximum amount of time that a server fetch operation will wait before returning
* (unless {@code minFetchSizeInBytes}) are available.
* In conjunction with {@link AbstractConfiguration#setMinBytes(int)},
* controls latency and throughput.
* Smaller intervals increase responsiveness, but may increase
* the number of poll operations, potentially increasing CPU
* consumption and reducing throughput.
* @return the maximum wait time for a fetch operation
*/
int getMaxWait();
/**
* The client name to be used throughout this connection.
* @return the client id for a connection
*/
String getClientId();
/**
* The buffer size for this client
* @return the buffer size
*/
int getBufferSize();
/**
* The socket timeout for this client
* @return the socket timeout
*/
int getSocketTimeout();
/**
* The retry backoff for this client
* @return the retry backoff
* @since 1.3
*/
int getBackOff();
/**
* The timeout on fetching metadata (e.g. partition leaders)
* @return the fetch metadata timeout
*/
int getFetchMetadataTimeout();
/**
* The list of seed broker addresses used by this Configuration.
* @return the broker addresses
*/
List<BrokerAddress> getBrokerAddresses();
/**
* A list of default partitions to perform operations on.
* @return the list of partitions
*/
List<Partition> getDefaultPartitions();
/**
* A default topic to perform operations on.
* @return a topic name
*/
String getDefaultTopic();
}

92
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Connection.java
View File

@@ -1,92 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import java.util.Map;
import kafka.api.OffsetRequest;
/**
* A connection to a Kafka broker.
*
* @author Marius Bogoevici
*/
public interface Connection {
/**
* Fetch data from a Kafka broker.
* @param fetchRequests a list of fetch operations
* @return message batches, indexed by partition
* @throws ConsumerException the ConsumerException if any underlying error
*/
Result<KafkaMessageBatch> fetch(FetchRequest... fetchRequests) throws ConsumerException;
/**
* Fetch an actual offset in the partition, immediately before the given reference time,
* or the smallest and largest value, respectively, if the special values -1
* ({@link OffsetRequest#LatestTime()}) and -2 ({@link OffsetRequest#EarliestTime()})
* are used . To be used to position the initial offset of a read operation.
* @param referenceTime The returned values will be before this time, if they exist. The special
* values -2 ({@link OffsetRequest#EarliestTime()}) and -1 ({@link OffsetRequest#LatestTime()}) are supported.
* @param partitions the offsets, indexed by {@link Partition}
* @return any errors, an empty {@link Result} in case of success
* @throws ConsumerException the ConsumerException if any underlying error
*/
Result<Long> fetchInitialOffset(long referenceTime, Partition... partitions) throws ConsumerException;
/**
* Fetch offsets from the native Kafka offset management system.
* @param consumerId the id of the consumer
* @param partitions the list of partitions whose offsets are queried for
* @return any errors, an empty {@link Result} in case of success
* @throws ConsumerException the ConsumerException if any underlying error
*/
Result<Long> fetchStoredOffsetsForConsumer(String consumerId, Partition... partitions) throws ConsumerException;
/**
* Update offsets in the native Kafka offset management system.
* @param consumerId the id of the consumer
* @param offsets the offsets, indexed by {@link Partition}
* @return any errors, an empty {@link Result} in case of success
* @throws ConsumerException the ConsumerException if any underlying error
*/
Result<Void> commitOffsetsForConsumer(String consumerId, Map<Partition, Long> offsets) throws ConsumerException;
/**
* Retrieve the leader broker addresses for all the partitions in the given topics.
* @param topics the topics whose partitions we query for
* @return broker addresses, indexed by {@link Partition}
* @throws ConsumerException the ConsumerException if any underlying error
* @deprecated as of 1.3, only {@link ConnectionFactory#getLeaders(Iterable)} should be used
*/
@Deprecated
Result<BrokerAddress> findLeaders(String... topics) throws ConsumerException;
/**
* The broker address for this consumer
* @return broker address
*/
BrokerAddress getBrokerAddress();
/**
* Closes the connection to the broker. No further operations are permitted.
*/
void close();
}

73
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConnectionFactory.java
View File

@@ -1,73 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import java.util.Collection;
import java.util.Map;
/**
* Creates Kafka connections and retrieves metadata for topics and partitions.
*
* @author Marius Bogoevici
*
*/
public interface ConnectionFactory {
/**
* Create a connection to a Kafka broker, caching it internally
* @param brokerAddress a broker address
* @return a working connection
*/
Connection connect(BrokerAddress brokerAddress);
/**
* Close the connection to the broker
* @param brokerAddress brokerAddress
*/
void disconnect(BrokerAddress brokerAddress);
/**
* Retrieve the leaders for a set of partitions.
* @param partitions whose leaders are queried
* @return the broker associated with the provided topic and partition
*/
Map<Partition, BrokerAddress> getLeaders(Iterable<Partition> partitions);
/**
* Return the leader for a single partition
* @param partition the partition whose leader is queried
* @return the leader's address
*/
BrokerAddress getLeader(Partition partition);
/**
* Refresh the cached metadata (i.e. leader topology and partitions). To be called when the topology changes
* are detected (i.e. brokers leave and/or partition leaders change) and that results in fetch errors,
* for instance.
* @param topics the topics for which to refresh the leaders
*/
void refreshMetadata(Collection<String> topics);
/**
* Retrieves the partitions of a given topic
* @param topic the topic to query for
* @return a list of partitions
*/
Collection<Partition> getPartitions(String topic);
}

41
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConsumerException.java
View File

@@ -1,41 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import kafka.javaapi.consumer.SimpleConsumer;
/**
* Wraps exceptions thrown by {@link SimpleConsumer} calls.
*
* Because the client is originally written in Scala, checked exceptions are missing from the method signatures
* and cannot be caught directly.
*
* @author Marius Bogoevici
*/
@SuppressWarnings("serial")
public class ConsumerException extends RuntimeException {
public ConsumerException(Throwable cause) {
super(cause);
}
public ConsumerException(String message) {
super(message);
}
}

324
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnection.java
View File

@@ -1,324 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import com.gs.collections.api.block.function.Function;
import com.gs.collections.api.block.function.Function2;
import com.gs.collections.api.tuple.Pair;
import com.gs.collections.impl.list.mutable.FastList;
import com.gs.collections.impl.tuple.Tuples;
import com.gs.collections.impl.utility.LazyIterate;
import com.gs.collections.impl.utility.MapIterate;
import kafka.api.FetchRequestBuilder;
import kafka.api.PartitionOffsetRequestInfo;
import kafka.cluster.Broker;
import kafka.common.ErrorMapping;
import kafka.common.OffsetAndMetadata;
import kafka.common.OffsetMetadataAndError;
import kafka.common.TopicAndPartition;
import kafka.javaapi.FetchResponse;
import kafka.javaapi.OffsetCommitRequest;
import kafka.javaapi.OffsetCommitResponse;
import kafka.javaapi.OffsetFetchRequest;
import kafka.javaapi.OffsetFetchResponse;
import kafka.javaapi.OffsetRequest;
import kafka.javaapi.OffsetResponse;
import kafka.javaapi.PartitionMetadata;
import kafka.javaapi.TopicMetadata;
import kafka.javaapi.TopicMetadataRequest;
import kafka.javaapi.TopicMetadataResponse;
import kafka.javaapi.consumer.SimpleConsumer;
import kafka.javaapi.message.ByteBufferMessageSet;
import kafka.message.MessageAndOffset;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.util.Assert;
/**
* A connection to a Kafka broker.
*
* @author Marius Bogoevici
*/
public class DefaultConnection implements Connection {
private static Log log = LogFactory.getLog(DefaultConnection.class);
private final AtomicInteger correlationIdCounter = new AtomicInteger(new Random(new Date().getTime()).nextInt());
private final SimpleConsumer simpleConsumer;
private final BrokerAddress brokerAddress;
private int minBytes;
private int maxWait;
public DefaultConnection(BrokerAddress brokerAddress, String clientId, int bufferSize, int soTimeout,
int minBytes, int maxWait) {
this.brokerAddress = brokerAddress;
this.minBytes = minBytes;
this.maxWait = maxWait;
this.simpleConsumer =
new SimpleConsumer(brokerAddress.getHost(), brokerAddress.getPort(), soTimeout, bufferSize, clientId);
}
/**
* @see Connection#getBrokerAddress()
*/
@Override
public BrokerAddress getBrokerAddress() {
return brokerAddress;
}
@Override
public void close() {
this.simpleConsumer.close();
}
/**
* Fetche data from Kafka.
* @return a combination of messages and errors, depending on whether the invocation was successful or not
* @throws ConsumerException the ConsumerException if any underlying error
*/
@Override
public Result<KafkaMessageBatch> fetch(FetchRequest... requests) throws ConsumerException {
FetchRequestBuilder fetchRequestBuilder = new FetchRequestBuilder();
for (FetchRequest request : requests) {
Partition partition = request.getPartition();
long offset = request.getOffset();
int maxSize = request.getMaxSizeInBytes();
fetchRequestBuilder.addFetch(partition.getTopic(), partition.getId(), offset, maxSize);
}
FetchResponse fetchResponse;
try {
fetchResponse = this.simpleConsumer.fetch(fetchRequestBuilder.maxWait(maxWait).minBytes(minBytes).build());
}
catch (Exception e) {
throw new ConsumerException(e);
}
ResultBuilder<KafkaMessageBatch> resultBuilder = new ResultBuilder<KafkaMessageBatch>();
for (final FetchRequest request : requests) {
Partition partition = request.getPartition();
if (log.isTraceEnabled()) {
log.trace("Reading from " + partition + "@" + request.getOffset());
}
short errorCode = fetchResponse.errorCode(partition.getTopic(), partition.getId());
if (ErrorMapping.NoError() == errorCode) {
ByteBufferMessageSet messageSet = fetchResponse.messageSet(partition.getTopic(), partition.getId());
List<KafkaMessage> kafkaMessages = LazyIterate.collect(messageSet,
new ConvertToKafkaMessageFunction(request)).toList();
long highWatermark = fetchResponse.highWatermark(partition.getTopic(), partition.getId());
resultBuilder.add(partition).withResult(new KafkaMessageBatch(partition, kafkaMessages, highWatermark));
}
else {
resultBuilder.add(partition).withError(errorCode);
}
}
return resultBuilder.build();
}
@Override
public Result<Long> fetchStoredOffsetsForConsumer(String consumerId, Partition... partitions) throws
ConsumerException {
FastList<TopicAndPartition> topicsAndPartitions = FastList.newList(Arrays.asList(partitions))
.collect(new ConvertToTopicAndPartitionFunction());
OffsetFetchRequest offsetFetchRequest = new OffsetFetchRequest(consumerId, topicsAndPartitions,
kafka.api.OffsetFetchRequest.CurrentVersion(), createCorrelationId(), simpleConsumer.clientId());
OffsetFetchResponse offsetFetchResponse = null;
try {
offsetFetchResponse = simpleConsumer.fetchOffsets(offsetFetchRequest);
}
catch (Exception e) {
throw new ConsumerException(e);
}
ResultBuilder<Long> resultBuilder = new ResultBuilder<Long>();
for (Partition partition : partitions) {
OffsetMetadataAndError offsetMetadataAndError
= offsetFetchResponse.offsets().get(new TopicAndPartition(partition.getTopic(), partition.getId()));
short errorCode = offsetMetadataAndError.error();
if (ErrorMapping.NoError() == errorCode) {
resultBuilder.add(partition).withResult(offsetMetadataAndError.offset());
}
else {
resultBuilder.add(partition).withError(errorCode);
}
}
return resultBuilder.build();
}
/**
* @see Connection#fetchInitialOffset(long, Partition...)
*/
@Override
public Result<Long> fetchInitialOffset(long referenceTime, Partition... partitions) throws
ConsumerException {
Assert.isTrue(partitions.length > 0, "Must provide at least one partition");
Map<TopicAndPartition, PartitionOffsetRequestInfo> infoMap = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
for (Partition partition : partitions) {
infoMap.put(new TopicAndPartition(partition.getTopic(), partition.getId()),
new PartitionOffsetRequestInfo(referenceTime, 1));
}
OffsetRequest offsetRequest =
new OffsetRequest(infoMap, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId());
OffsetResponse offsetResponse = null;
try {
offsetResponse = simpleConsumer.getOffsetsBefore(offsetRequest);
}
catch (Exception e) {
throw new ConsumerException(e);
}
ResultBuilder<Long> resultBuilder = new ResultBuilder<Long>();
for (Partition partition : partitions) {
short errorCode = offsetResponse.errorCode(partition.getTopic(), partition.getId());
if (ErrorMapping.NoError() == errorCode) {
long[] offsets = offsetResponse.offsets(partition.getTopic(), partition.getId());
if (offsets.length == 0) {
// normally, we shouldn't get here - either an offset has been returned,
// or an error. However, in case something went wrong, the check protects against an
// ArrayIndexOutOfBoundsException
throw new ConsumerException("Inconsistent response: no error has been returned, " +
"but no offsets either");
}
resultBuilder.add(partition).withResult(offsets[0]);
}
else {
resultBuilder.add(partition).withError(errorCode);
}
}
return resultBuilder.build();
}
@Override
public Result<Void> commitOffsetsForConsumer(String consumerId, Map<Partition, Long> offsets)
throws ConsumerException {
Map<TopicAndPartition, OffsetAndMetadata> requestInfo =
MapIterate.collect(offsets, new CreateRequestInfoMapEntryFunction());
OffsetCommitResponse offsetCommitResponse = null;
try {
offsetCommitResponse = simpleConsumer.commitOffsets(
new OffsetCommitRequest(consumerId, requestInfo, createCorrelationId(),
simpleConsumer.clientId(), kafka.api.OffsetCommitRequest.CurrentVersion()));
}
catch (Exception e) {
throw new ConsumerException(e);
}
ResultBuilder<Void> resultBuilder = new ResultBuilder<Void>();
for (TopicAndPartition topicAndPartition : requestInfo.keySet()) {
if (offsetCommitResponse.errors().containsKey(topicAndPartition)) {
Partition partition = new Partition(topicAndPartition.topic(), topicAndPartition.partition());
resultBuilder.add(partition).withError((Short) offsetCommitResponse.errors().get(topicAndPartition));
}
}
return resultBuilder.build();
}
/**
* @see Connection#findLeaders(String...)
*/
@Override
@Deprecated
public Result<BrokerAddress> findLeaders(String... topics) throws ConsumerException {
TopicMetadataRequest topicMetadataRequest =
new TopicMetadataRequest(Arrays.asList(topics), createCorrelationId());
TopicMetadataResponse topicMetadataResponse = null;
try {
topicMetadataResponse = simpleConsumer.send(topicMetadataRequest);
}
catch (Exception e) {
throw new ConsumerException(e);
}
ResultBuilder<BrokerAddress> resultBuilder = new ResultBuilder<BrokerAddress>();
for (TopicMetadata topicMetadata : topicMetadataResponse.topicsMetadata()) {
if (topicMetadata.errorCode() != ErrorMapping.NoError()) {
resultBuilder.add(new Partition(topicMetadata.topic(), -1)).withError(topicMetadata.errorCode());
}
else {
for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) {
Partition partition = new Partition(topicMetadata.topic(), partitionMetadata.partitionId());
if (ErrorMapping.NoError() == partitionMetadata.errorCode()) {
Broker leader = partitionMetadata.leader();
BrokerAddress result = new BrokerAddress(leader.host(), leader.port());
resultBuilder.add(partition).withResult(result);
}
else {
resultBuilder.add(partition).withError(partitionMetadata.errorCode());
}
}
}
}
return resultBuilder.build();
}
/**
* Creat a pseudo-unique correlation id for requests and responses
* @return correlation id
*/
private Integer createCorrelationId() {
return correlationIdCounter.incrementAndGet();
}
@SuppressWarnings("serial")
private static class ConvertToKafkaMessageFunction implements Function<MessageAndOffset, KafkaMessage> {
private final FetchRequest request;
public ConvertToKafkaMessageFunction(FetchRequest request) {
this.request = request;
}
@Override
public KafkaMessage valueOf(MessageAndOffset messageAndOffset) {
return new KafkaMessage(messageAndOffset.message(),
new KafkaMessageMetadata(request.getPartition(), messageAndOffset.offset(),
messageAndOffset.nextOffset()));
}
}
@SuppressWarnings("serial")
private static class ConvertToTopicAndPartitionFunction implements Function<Partition, TopicAndPartition> {
@Override
public TopicAndPartition valueOf(Partition partition) {
return new TopicAndPartition(partition.getTopic(), partition.getId());
}
}
@SuppressWarnings("serial")
private static class CreateRequestInfoMapEntryFunction
implements Function2<Partition, Long, Pair<TopicAndPartition, OffsetAndMetadata>> {
@Override
public Pair<TopicAndPartition, OffsetAndMetadata> value(Partition partition, Long offset) {
return Tuples.pair(new TopicAndPartition(partition.getTopic(), partition.getId()),
new OffsetAndMetadata(offset, OffsetAndMetadata.NoMetadata(), OffsetAndMetadata.InvalidTime()));
}
}
}

276
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnectionFactory.java
View File

@@ -1,276 +0,0 @@
/*
* Copyright 2015-2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.util.Assert;
import com.gs.collections.api.block.function.Function;
import com.gs.collections.api.block.predicate.Predicate;
import com.gs.collections.api.partition.PartitionIterable;
import com.gs.collections.impl.block.factory.Functions;
import com.gs.collections.impl.map.mutable.UnifiedMap;
import com.gs.collections.impl.utility.Iterate;
import com.gs.collections.impl.utility.ListIterate;
import kafka.client.ClientUtils$;
import kafka.cluster.Broker;
import kafka.common.ErrorMapping;
import kafka.javaapi.TopicMetadata;
import kafka.javaapi.TopicMetadataResponse;
import scala.collection.JavaConversions;
import scala.collection.Seq;
/**
* Default implementation of {@link ConnectionFactory}
*
* @author Marius Bogoevici
* @author Artem Bilan
*/
public class DefaultConnectionFactory implements InitializingBean, ConnectionFactory, DisposableBean {
private final static Log log = LogFactory.getLog(DefaultConnectionFactory.class);
public static final Predicate<TopicMetadata> errorlessTopicMetadataPredicate = new ErrorlessTopicMetadataPredicate();
private final GetBrokersByPartitionFunction getBrokersByPartitionFunction = new GetBrokersByPartitionFunction();
private final Configuration configuration;
private final AtomicReference<MetadataCache> metadataCacheHolder = new AtomicReference<MetadataCache>(
new MetadataCache(Collections.<TopicMetadata>emptySet()));
private final ReadWriteLock lock = new ReentrantReadWriteLock();
private final UnifiedMap<BrokerAddress, Connection> kafkaBrokersCache = UnifiedMap.newMap();
public DefaultConnectionFactory(Configuration configuration) {
this.configuration = configuration;
}
public Configuration getConfiguration() {
return this.configuration;
}
@Override
public void afterPropertiesSet() throws Exception {
Assert.notNull(this.configuration, "Kafka configuration cannot be empty");
}
@Override
public void destroy() throws Exception {
for (Connection connection : this.kafkaBrokersCache) {
connection.close();
}
}
/**
* @see ConnectionFactory#getLeaders(Iterable)
*/
@Override
public Map<Partition, BrokerAddress> getLeaders(Iterable<Partition> partitions) {
return Iterate.toMap(partitions, Functions.<Partition>getPassThru(), getBrokersByPartitionFunction);
}
/**
* @see ConnectionFactory#getLeader(Partition)
*/
@Override
public BrokerAddress getLeader(Partition partition) {
BrokerAddress leader = null;
try {
this.lock.readLock().lock();
leader = getMetadataCache().getLeader(partition);
}
finally {
this.lock.readLock().unlock();
}
if (leader == null) {
try {
this.lock.writeLock().lock();
// double lock check
leader = getMetadataCache().getLeader(partition);
if (leader == null) {
refreshMetadata(Collections.singleton(partition.getTopic()));
leader = getMetadataCache().getLeader(partition);
}
}
finally {
this.lock.writeLock().unlock();
}
}
if (leader == null) {
throw new PartitionNotFoundException(partition);
}
return leader;
}
/**
* @see ConnectionFactory#connect(BrokerAddress)
*/
@Override
public Connection connect(BrokerAddress brokerAddress) {
Connection connection = null;
try {
this.lock.readLock().lock();
connection = this.kafkaBrokersCache.get(brokerAddress);
}
finally {
this.lock.readLock().unlock();
}
if (connection == null) {
try {
this.lock.writeLock().lock();
connection = this.kafkaBrokersCache.get(brokerAddress);
if (connection == null) {
connection = new DefaultConnection(brokerAddress,
DefaultConnectionFactory.this.configuration.getClientId(),
DefaultConnectionFactory.this.configuration.getBufferSize(),
DefaultConnectionFactory.this.configuration.getSocketTimeout(),
DefaultConnectionFactory.this.configuration.getMinBytes(),
DefaultConnectionFactory.this.configuration.getMaxWait());
kafkaBrokersCache.put(brokerAddress, connection);
}
}
finally {
this.lock.writeLock().unlock();
}
}
return connection;
}
/**
* @see ConnectionFactory#refreshMetadata(Collection)
*/
@Override
public void refreshMetadata(Collection<String> topics) {
try {
this.lock.writeLock().lock();
String brokerAddressesAsString = ListIterate
.collect(this.configuration.getBrokerAddresses(), Functions.getToString()).makeString(",");
Seq<Broker> brokers = null;
try {
brokers = ClientUtils$.MODULE$.parseBrokerList(brokerAddressesAsString);
}
catch (Exception e) {
throw new IllegalStateException("Can not parse Kafka Brokers for: [" + brokerAddressesAsString + "]", e);
}
TopicMetadataResponse topicMetadataResponse = new TopicMetadataResponse(ClientUtils$.MODULE$
.fetchTopicMetadata(JavaConversions.asScalaSet(new HashSet<>(topics)),
brokers,
this.configuration.getClientId(), this.configuration.getFetchMetadataTimeout(), 0));
PartitionIterable<TopicMetadata> selectWithoutErrors = Iterate
.partition(topicMetadataResponse.topicsMetadata(), errorlessTopicMetadataPredicate);
this.metadataCacheHolder.set(this.metadataCacheHolder.get().merge(selectWithoutErrors.getSelected()));
if (log.isInfoEnabled()) {
for (TopicMetadata topicMetadata : selectWithoutErrors.getRejected()) {
log.info(String.format("No metadata could be retrieved for '%s'", topicMetadata.topic()),
ErrorMapping.exceptionFor(topicMetadata.errorCode()));
}
}
}
finally {
this.lock.writeLock().unlock();
}
}
@Override
public void disconnect(BrokerAddress brokerAddress) {
try {
this.lock.writeLock().lock();
Connection connection = this.kafkaBrokersCache.get(brokerAddress);
if (connection != null) {
connection.close();
}
}
finally {
this.lock.writeLock().unlock();
}
}
/**
* @see ConnectionFactory#getPartitions(String)
*/
@Override
public Collection<Partition> getPartitions(String topic) {
// first, we try to read the topic from the cache. We use the read lock to block if a write is in progress
Collection<Partition> returnedPartitions = null;
try {
this.lock.readLock().lock();
returnedPartitions = getMetadataCache().getPartitions(topic);
}
finally {
this.lock.readLock().unlock();
}
// if we got here, it means that the data was not available, we should try a refresh. The lock is reentrant
// so we will not block ourselves
if (returnedPartitions == null) {
try {
this.lock.writeLock().lock();
// double lock check
returnedPartitions = getMetadataCache().getPartitions(topic);
if (returnedPartitions == null) {
refreshMetadata(Collections.singleton(topic));
// if data is not available after refreshing, it means that the topic was not found
returnedPartitions = getMetadataCache().getPartitions(topic);
}
}
finally {
lock.writeLock().unlock();
}
}
if (returnedPartitions == null) {
throw new TopicNotFoundException(topic);
}
return returnedPartitions;
}
private MetadataCache getMetadataCache() {
return this.metadataCacheHolder.get();
}
@SuppressWarnings("serial")
private static class ErrorlessTopicMetadataPredicate implements Predicate<TopicMetadata> {
@Override
public boolean accept(TopicMetadata topicMetadata) {
return topicMetadata.errorCode() == ErrorMapping.NoError();
}
}
@SuppressWarnings("serial")
private class GetBrokersByPartitionFunction implements Function<Partition, BrokerAddress> {
@Override
public BrokerAddress valueOf(Partition partition) {
return metadataCacheHolder.get().getLeader(partition);
}
}
}

62
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/FetchRequest.java
View File

@@ -1,62 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
/**
* Encapsulates a request for fetching messages from the server.
*
* @author Marius Bogoevici
*/
public class FetchRequest {
private Partition partition;
private long offset;
private int maxSizeInBytes;
public FetchRequest(Partition Partition, long offset, int maxSizeInBytes) {
this.partition = Partition;
this.offset = offset;
this.maxSizeInBytes = maxSizeInBytes;
}
public Partition getPartition() {
return partition;
}
public void setPartition(Partition partition) {
this.partition = partition;
}
public long getOffset() {
return offset;
}
public void setOffset(long offset) {
this.offset = offset;
}
public int getMaxSizeInBytes() {
return maxSizeInBytes;
}
public void setMaxSizeInBytes(int maxSizeInBytes) {
this.maxSizeInBytes = maxSizeInBytes;
}
}

70
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java
View File

@@ -1,70 +0,0 @@
/*
* Copyright 2002-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import kafka.api.OffsetRequest;
/**
* Kafka adapter specific message headers.
*
* @author Soby Chacko
* @author Marius Bogoevici
* @since 0.5
*/
public abstract class KafkaConsumerDefaults {
//High level consumer
public static final String GROUP_ID = "groupid";
public static final int SOCKET_TIMEOUT_INT = 30000;
public static final String SOCKET_TIMEOUT = Integer.toString(SOCKET_TIMEOUT_INT);
public static final int SOCKET_BUFFER_SIZE_INT = 64*1024;
public static final String SOCKET_BUFFER_SIZE = Integer.toString(SOCKET_BUFFER_SIZE_INT);
public static final int FETCH_SIZE_INT = 300 * 1024;
public static final String FETCH_SIZE = Integer.toString(FETCH_SIZE_INT);
public static final int BACKOFF_INCREMENT_INT = 1000;
public static final String BACKOFF_INCREMENT = Integer.toString(BACKOFF_INCREMENT_INT);
public static final String QUEUED_CHUNKS_MAX = "100";
public static final String AUTO_COMMIT_ENABLE = "true";
public static final String AUTO_COMMIT_INTERVAL = "10000";
public static final String AUTO_OFFSET_RESET = "smallest";
//Overriding the default value of -1, which will make the consumer to wait indefinitely
public static final String CONSUMER_TIMEOUT = "5000";
public static final String REBALANCE_RETRIES_MAX = "4";
public static final int MIN_FETCH_BYTES = 1;
public static final int MAX_WAIT_TIME_IN_MS = 100;
public static final long DEFAULT_OFFSET_RESET = OffsetRequest.EarliestTime();
public static final int FETCH_METADATA_TIMEOUT = 10000;
}

53
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessage.java
View File

@@ -1,53 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import kafka.message.Message;
import org.springframework.util.ObjectUtils;
/**
* Wrapper around a {@link Message kafka message} and its {@link KafkaMessageMetadata metadata}.
*
* @author Marius Bogoevici
*/
public class KafkaMessage {
private final Message message;
private final KafkaMessageMetadata metadata;
public KafkaMessage(Message message, KafkaMessageMetadata metadata) {
this.message = message;
this.metadata = metadata;
}
public Message getMessage() {
return message;
}
public KafkaMessageMetadata getMetadata() {
return metadata;
}
@Override
public String toString() {
return "KafkaMessage [" + ObjectUtils.nullSafeToString(message) + ", "
+ ObjectUtils.nullSafeToString(metadata.toString()) + "]";
}
}

64
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageBatch.java
View File

@@ -1,64 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import java.util.List;
/**
* A group of {@link KafkaMessage}s retrieved by a fetch operation
*
* @author Marius Bogoevici
*/
public class KafkaMessageBatch {
private Partition partition;
private List<KafkaMessage> messages;
private long highWatermark;
public KafkaMessageBatch(Partition partition, List<KafkaMessage> messages, long highWatermark) {
this.partition = partition;
this.messages = messages;
this.highWatermark = highWatermark;
}
public Partition getPartition() {
return partition;
}
public void setPartition(Partition partition) {
this.partition = partition;
}
public List<KafkaMessage> getMessages() {
return messages;
}
public void setMessages(List<KafkaMessage> messages) {
this.messages = messages;
}
public long getHighWatermark() {
return highWatermark;
}
public void setHighWatermark(long highWatermark) {
this.highWatermark = highWatermark;
}
}

65
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageMetadata.java
View File

@@ -1,65 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import kafka.message.Message;
import org.springframework.util.Assert;
/**
* Metadata for a Kafka {@link Message}.
*
* @author Marius Bogoevici
*/
public class KafkaMessageMetadata {
private final long offset;
private final long nextOffset;
private final Partition partition;
public KafkaMessageMetadata(Partition partition, long offset, long nextOffset) {
Assert.notNull(partition);
Assert.isTrue(offset >= 0);
Assert.isTrue(nextOffset >= 0);
this.offset = offset;
this.nextOffset = nextOffset;
this.partition = partition;
}
public Partition getPartition() {
return partition;
}
public long getOffset() {
return offset;
}
public long getNextOffset() {
return nextOffset;
}
@Override
public String toString() {
return "KafkaMessageMetadata [" +
"offset=" + offset +
", nextOffset=" + nextOffset +
", " + partition.toString();
}
}

31
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaOperations.java
View File

@@ -1,31 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
/**
* @author Marius Bogoevici
*/
public interface KafkaOperations {
/**
* Receive data from the server.
* @param messageFetchRequests the set of FetchRequest
* @return the Result with KafkaMessageBatch
*/
Result<KafkaMessageBatch> receive(Iterable<FetchRequest> messageFetchRequests);
}

65
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaTemplate.java
View File

@@ -1,65 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import com.gs.collections.api.block.function.Function;
import com.gs.collections.api.list.MutableList;
import com.gs.collections.impl.list.mutable.FastList;
/**
* A template for executing high-level operations on a set of Kafka brokers.
*
* @author Marius Bogoevici
*/
public class KafkaTemplate implements KafkaOperations {
private final ConnectionFactory connectionFactory;
private FetchRequestToLeaderBrokerAddress fetchRequestToLeaderBrokerAddress = new FetchRequestToLeaderBrokerAddress();
public KafkaTemplate(ConnectionFactory connectionFactory) {
this.connectionFactory = connectionFactory;
}
public ConnectionFactory getConnectionFactory() {
return connectionFactory;
}
@Override
public Result<KafkaMessageBatch> receive(Iterable<FetchRequest> messageFetchRequests) {
FastList<FetchRequest> requestList = FastList.newList(messageFetchRequests);
MutableList<BrokerAddress> distinctBrokerAddresses =
requestList.collect(fetchRequestToLeaderBrokerAddress).distinct();
if (distinctBrokerAddresses.size() != 1) {
throw new IllegalArgumentException("All messages must be fetched from the same broker");
}
return connectionFactory.connect(distinctBrokerAddresses.getFirst())
.fetch(requestList.toTypedArray(FetchRequest.class));
}
@SuppressWarnings("serial")
private class FetchRequestToLeaderBrokerAddress implements Function<FetchRequest, BrokerAddress> {
@Override
public BrokerAddress valueOf(FetchRequest fetchRequest) {
return connectionFactory.getLeader(fetchRequest.getPartition());
}
}
}

111
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/MetadataCache.java
View File

@@ -1,111 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import java.util.Collection;
import java.util.Map;
import com.gs.collections.api.block.function.Function;
import com.gs.collections.impl.map.mutable.UnifiedMap;
import com.gs.collections.impl.utility.Iterate;
import kafka.javaapi.PartitionMetadata;
import kafka.javaapi.TopicMetadata;
/**
* Immutable store for the partition/broker mapping
*
* @author Marius Bogoevici
*/
class MetadataCache {
public static final GetTopicNameFunction getTopicNameFunction = new GetTopicNameFunction();
public static final ToIndexedPartitionMetadataFunction toIndexedPartitionMetadataFunction =
new ToIndexedPartitionMetadataFunction();
private final Map<String, Map<Partition, PartitionMetadata>> metadatasByTopic;
public MetadataCache(Iterable<TopicMetadata> topicMetadatas) {
UnifiedMap<String, Map<Partition, PartitionMetadata>> topicData = UnifiedMap.newMap();
for (TopicMetadata topicMetadata : topicMetadatas) {
Map<Partition, PartitionMetadata> partitionData = toPartitionMetadataMap(topicMetadata);
topicData.put(topicMetadata.topic(), partitionData);
}
this.metadatasByTopic = topicData.toImmutable().castToMap();
}
private MetadataCache(Map<String, Map<Partition, PartitionMetadata>> metadatasByTopic) {
this.metadatasByTopic = metadatasByTopic;
}
public MetadataCache merge(final Iterable<TopicMetadata> topicMetadatas) {
UnifiedMap<String, Map<Partition, PartitionMetadata>> unifiedMap = UnifiedMap.newMap(this.metadatasByTopic);
return new MetadataCache(Iterate.addToMap(topicMetadatas, getTopicNameFunction,
toIndexedPartitionMetadataFunction, unifiedMap).toImmutable().castToMap());
}
public Collection<Partition> getPartitions(String topic) {
if (this.metadatasByTopic.containsKey(topic)) {
return this.metadatasByTopic.get(topic).keySet();
}
else {
return null;
}
}
public BrokerAddress getLeader(Partition partition) {
if (!this.metadatasByTopic.containsKey(partition.getTopic())) {
return null;
}
Map<Partition, PartitionMetadata> partitionMetadatasForTopic = this.metadatasByTopic.get(partition.getTopic());
if (!partitionMetadatasForTopic.containsKey(partition)) {
return null;
}
return new BrokerAddress(partitionMetadatasForTopic.get(partition).leader());
}
private static Map<Partition, PartitionMetadata> toPartitionMetadataMap(TopicMetadata topicMetadata) {
Map<Partition, PartitionMetadata> partitionData = UnifiedMap.newMap();
for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) {
partitionData.put(new Partition(topicMetadata.topic(), partitionMetadata.partitionId()), partitionMetadata);
}
return partitionData;
}
@SuppressWarnings("serial")
private static class GetTopicNameFunction implements Function<TopicMetadata, String> {
@Override
public String valueOf(TopicMetadata object) {
return object.topic();
}
}
@SuppressWarnings("serial")
private static class ToIndexedPartitionMetadataFunction implements
Function<TopicMetadata, Map<Partition, PartitionMetadata>> {
@Override
public Map<Partition, PartitionMetadata> valueOf(TopicMetadata object) {
return toPartitionMetadataMap(object);
}
}
}

82
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Partition.java
View File

@@ -1,82 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import org.springframework.util.Assert;
/**
* A reference to a Kafka partition, with both topic and partition id
*
* @author Marius Bogoevici
*/
public class Partition {
private String topic;
private int id;
public Partition(String topic, int id) {
Assert.hasText(topic, "Topic name cannot be empty");
Assert.isTrue(id >= 0, "Partition id must be greater than or equal to 0");
this.topic = topic;
this.id = id;
}
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
@Override
public int hashCode() {
int result = topic.hashCode();
result = 31 * result + id;
return result;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Partition partition = (Partition) o;
if (id != partition.id) {
return false;
}
return topic.equals(partition.topic);
}
@Override
public String toString() {
return "Partition[" + "topic='" + topic + '\'' + ", id=" + id + ']';
}
}

30
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/PartitionNotFoundException.java
View File

@@ -1,30 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
/**
* @author Marius Bogoevici
*/
@SuppressWarnings("serial")
public class PartitionNotFoundException extends ConsumerException {
public PartitionNotFoundException(Partition partition) {
super(String.format("Partition [%s,%d] has no leader or has not been found",
partition.getTopic(), partition.getId()));
}
}

64
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Result.java
View File

@@ -1,64 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import java.util.Collections;
import java.util.Map;
/**
* The result of a Kafka SimpleConsumer operation
*
* @author Marius Bogoevici
*/
public class Result<T> {
private final Map<Partition, T> results;
private final Map<Partition, Short> errors;
Result(Map<Partition, T> results, Map<Partition, Short> errors) {
this.results = Collections.unmodifiableMap(results);
this.errors = Collections.unmodifiableMap(errors);
}
public Map<Partition, T> getResults() {
return results;
}
public T getResult(Partition partition) throws IllegalArgumentException {
if (this.results.containsKey(partition)) {
return this.results.get(partition);
}
else {
throw new IllegalArgumentException(" No result received for " + partition.toString());
}
}
public Map<Partition, Short> getErrors() {
return errors;
}
public short getError(Partition partition) throws IllegalArgumentException {
if (this.getErrors().containsKey(partition)) {
return this.getErrors().get(partition);
}
else {
throw new IllegalArgumentException("No error received for " + partition.toString());
}
}
}

74
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ResultBuilder.java
View File

@@ -1,74 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import java.util.HashMap;
import java.util.Map;
/**
* Utility class for building a {@link Result}
*
* @author Marius Bogoevici
*/
class ResultBuilder<T> {
private Map<Partition, T> result;
private Map<Partition, Short> errors;
public ResultBuilder() {
this.result = new HashMap<Partition, T>();
this.errors = new HashMap<Partition, Short>();
}
public KafkaPartitionResultHolder add(Partition Partition) {
return new KafkaPartitionResultHolder(Partition);
}
public Result<T> build() {
return new Result<T>(result, errors);
}
class KafkaPartitionResultHolder {
private Partition partition;
public KafkaPartitionResultHolder(Partition Partition) {
this.partition = Partition;
}
public ResultBuilder<T> withResult(T result) {
if (ResultBuilder.this.errors.containsKey(partition)) {
throw new IllegalArgumentException("A KafkaResult cannot contain both an error " +
"and a result for the same topic and partition");
}
ResultBuilder.this.result.put(partition, result);
return ResultBuilder.this;
}
public ResultBuilder<T> withError(short error) {
if (ResultBuilder.this.result.containsKey(partition)) {
throw new IllegalArgumentException("A FetchResult cannot contain both an error " +
"and a MessageSet for the same topic and partition");
}
ResultBuilder.this.errors.put(partition, error);
return ResultBuilder.this;
}
}
}

29
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/TopicNotFoundException.java
View File

@@ -1,29 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
/**
* @author Marius Bogoevici
*/
@SuppressWarnings("serial")
public class TopicNotFoundException extends ConsumerException {
public TopicNotFoundException(String topicName) {
super(String.format("No topic named '%s' found", topicName));
}
}

106
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConfiguration.java
View File

@@ -1,106 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
import java.util.List;
import org.I0Itec.zkclient.ZkClient;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.BeanInitializationException;
import org.springframework.integration.kafka.support.ZookeeperConnect;
import com.gs.collections.api.block.function.Function;
import com.gs.collections.impl.list.mutable.FastList;
import kafka.cluster.Broker;
import kafka.utils.ZKStringSerializer$;
import kafka.utils.ZkUtils$;
import scala.collection.JavaConversions;
import scala.collection.Seq;
/**
* Kafka {@link Configuration} that uses a ZooKeeper connection for retrieving the list of seed brokers.
*
* @author Marius Bogoevici
*/
public class ZookeeperConfiguration extends AbstractConfiguration {
private final static Log log = LogFactory.getLog(ZookeeperConfiguration.class);
public static final BrokerToBrokerAddressFunction brokerToBrokerAddressFunction = new BrokerToBrokerAddressFunction();
private String zookeeperServers;
private int sessionTimeout;
private int connectionTimeout;
public ZookeeperConfiguration(String zookeeperConnectionString) {
this(new ZookeeperConnect(zookeeperConnectionString));
}
public ZookeeperConfiguration(ZookeeperConnect zookeeperConnect) {
this.zookeeperServers = zookeeperConnect.getZkConnect();
try {
this.sessionTimeout = Integer.parseInt(zookeeperConnect.getZkSessionTimeout());
}
catch (NumberFormatException e) {
throw new BeanInitializationException("Cannot parse session timeout:", e);
}
try {
this.connectionTimeout = Integer.parseInt(zookeeperConnect.getZkConnectionTimeout());
}
catch (NumberFormatException e) {
throw new BeanInitializationException("Cannot parse connection timeout:", e);
}
}
@Override
protected List<BrokerAddress> doGetBrokerAddresses() {
ZkClient zkClient = null;
try {
zkClient = new ZkClient(this.zookeeperServers, this.sessionTimeout, this.connectionTimeout,
ZKStringSerializer$.MODULE$);
Seq<Broker> allBrokersInCluster = ZkUtils$.MODULE$.getAllBrokersInCluster(zkClient);
FastList<Broker> brokers = FastList.newList(JavaConversions.asJavaCollection(allBrokersInCluster));
return brokers.collect(brokerToBrokerAddressFunction);
}
finally {
if (zkClient != null) {
try {
zkClient.close();
}
catch (Exception e) {
log.error("Cannot close Zookeeper client: ", e);
}
}
}
}
@SuppressWarnings("serial")
private static class BrokerToBrokerAddressFunction implements Function<Broker, BrokerAddress> {
@Override
public BrokerAddress valueOf(Broker broker) {
return new BrokerAddress(broker.host(), broker.port());
}
}
}

32
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConnectDefaults.java
View File

@@ -1,32 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
/**
*
* @author Soby Chacko
* @since 0.5
*
*/
public final class ZookeeperConnectDefaults {
public static final String ZK_CONNECT = "localhost:2181";
public static final String ZK_CONNECTION_TIMEOUT = "6000";
public static final String ZK_SESSION_TIMEOUT = "6000";
public static final String ZK_SYNC_TIME = "2000";
private ZookeeperConnectDefaults() {
}
}

4
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/package-info.java
View File

@@ -1,4 +0,0 @@
/**
* Provides core classes of the Kafka module.
*/
package org.springframework.integration.kafka.core;

49
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaHighLevelConsumerMessageSource.java
View File

@@ -1,49 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.inbound;
import java.util.List;
import java.util.Map;
import org.springframework.integration.context.IntegrationObjectSupport;
import org.springframework.integration.core.MessageSource;
import org.springframework.messaging.Message;
/**
* @author Soby Chacko
* @since 0.5
* @deprecated since 1.3 in favor of {@link KafkaMessageDrivenChannelAdapter}
*/
@Deprecated
@SuppressWarnings("deprecation")
public class KafkaHighLevelConsumerMessageSource<K,V> extends IntegrationObjectSupport implements MessageSource<Map<String, Map<Integer, List<Object>>>> {
private final org.springframework.integration.kafka.support.KafkaConsumerContext<K,V> kafkaConsumerContext;
public KafkaHighLevelConsumerMessageSource(final org.springframework.integration.kafka.support.KafkaConsumerContext<K,V> kafkaConsumerContext) {
this.kafkaConsumerContext = kafkaConsumerContext;
}
@Override
public Message<Map<String, Map<Integer, List<Object>>>> receive() {
return kafkaConsumerContext.receive();
}
@Override
public String getComponentType() {
return "kafka:inbound-channel-adapter";
}
}

135
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaMessageDrivenChannelAdapter.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2015 the original author or authors.
* Copyright 2015-2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,20 +18,19 @@ package org.springframework.integration.kafka.inbound;
import java.util.Map;
import kafka.serializer.Decoder;
import kafka.serializer.DefaultDecoder;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.integration.context.OrderlyShutdownCapable;
import org.springframework.integration.endpoint.MessageProducerSupport;
import org.springframework.integration.kafka.core.KafkaMessageMetadata;
import org.springframework.integration.kafka.listener.AbstractDecodingAcknowledgingMessageListener;
import org.springframework.integration.kafka.listener.AbstractDecodingMessageListener;
import org.springframework.integration.kafka.listener.Acknowledgment;
import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer;
import org.springframework.integration.kafka.support.KafkaHeaders;
import org.springframework.integration.support.DefaultMessageBuilderFactory;
import org.springframework.integration.support.MessageBuilderFactory;
import org.springframework.integration.support.MutableMessageBuilderFactory;
import org.springframework.kafka.listener.AbstractMessageListenerContainer;
import org.springframework.kafka.listener.AbstractMessageListenerContainer.AckMode;
import org.springframework.kafka.listener.AcknowledgingMessageListener;
import org.springframework.kafka.listener.MessageListener;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.kafka.support.KafkaHeaders;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageHeaders;
import org.springframework.messaging.support.MessageBuilder;
@@ -39,14 +38,13 @@ import org.springframework.util.Assert;
/**
* @author Marius Bogoevici
* @author Gary Russell
*
* TODO: Use the MessagingMessageConverter from spring-kafka
*/
public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport implements OrderlyShutdownCapable {
public class KafkaMessageDrivenChannelAdapter<K, V> extends MessageProducerSupport implements OrderlyShutdownCapable {
private final KafkaMessageListenerContainer messageListenerContainer;
private Decoder<?> keyDecoder = new DefaultDecoder(null);
private Decoder<?> payloadDecoder = new DefaultDecoder(null);
private final AbstractMessageListenerContainer<K, V> messageListenerContainer;
private boolean generateMessageId = false;
@@ -54,40 +52,19 @@ public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport imp
private boolean useMessageBuilderFactory = false;
private boolean autoCommitOffset = true;
public KafkaMessageDrivenChannelAdapter(KafkaMessageListenerContainer messageListenerContainer) {
Assert.notNull(messageListenerContainer);
Assert.isNull(messageListenerContainer.getMessageListener());
public KafkaMessageDrivenChannelAdapter(AbstractMessageListenerContainer<K, V> messageListenerContainer) {
Assert.notNull(messageListenerContainer, "messageListenerContainer is required");
Assert.isNull(messageListenerContainer.getMessageListener(), "Container must not already have a listener");
this.messageListenerContainer = messageListenerContainer;
this.messageListenerContainer.setAutoStartup(false);
}
public void setKeyDecoder(Decoder<?> keyDecoder) {
this.keyDecoder = keyDecoder;
}
public void setPayloadDecoder(Decoder<?> payloadDecoder) {
this.payloadDecoder = payloadDecoder;
}
/**
* Automatically commit the offsets when 'true'. When 'false', the
* adapter inserts a 'kafka_acknowledgment` header allowing the user to manually
* commit the offset using the {@link Acknowledgment#acknowledge()} method.
* Default 'true'.
* @param autoCommitOffset false to not auto-commit (default true).
*/
public void setAutoCommitOffset(boolean autoCommitOffset) {
this.autoCommitOffset = autoCommitOffset;
}
/**
* Generate {@link Message} {@code ids} for produced messages.
* If set to {@code false}, will try to use a default value. By default set to {@code false}.
* Note that this option is only guaranteed to work when
* {@link #setUseMessageBuilderFactory(boolean) useMessageBuilderFactory} is false (default).
* If the latter is set to {@code true}, then some {@link MessageBuilderFactory} implementations such as
* Generate {@link Message} {@code ids} for produced messages. If set to {@code false}
* , will try to use a default value. By default set to {@code false}. Note that this
* option is only guaranteed to work when {@link #setUseMessageBuilderFactory(boolean)
* useMessageBuilderFactory} is false (default). If the latter is set to {@code true},
* then some {@link MessageBuilderFactory} implementations such as
* {@link DefaultMessageBuilderFactory} may ignore it.
* @param generateMessageId true if a message id should be generated
* @since 1.1
@@ -97,11 +74,11 @@ public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport imp
}
/**
* Generate {@code timestamp} for produced messages. If set to {@code false}, -1 is used instead.
* By default set to {@code false}.
* Note that this option is only guaranteed to work when
* {@link #setUseMessageBuilderFactory(boolean) useMessageBuilderFactory} is false (default).
* If the latter is set to {@code true}, then some {@link MessageBuilderFactory} implementations such as
* Generate {@code timestamp} for produced messages. If set to {@code false}, -1 is
* used instead. By default set to {@code false}. Note that this option is only
* guaranteed to work when {@link #setUseMessageBuilderFactory(boolean)
* useMessageBuilderFactory} is false (default). If the latter is set to {@code true},
* then some {@link MessageBuilderFactory} implementations such as
* {@link DefaultMessageBuilderFactory} may ignore it.
* @param generateTimestamp true if a timestamp should be generated
* @since 1.1
@@ -111,9 +88,10 @@ public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport imp
}
/**
* Use the {@link MessageBuilderFactory} returned by {@link #getMessageBuilderFactory()} to create messages.
* @param useMessageBuilderFactory true if the {@link MessageBuilderFactory} returned by
* {@link #getMessageBuilderFactory()} should be used.
* Use the {@link MessageBuilderFactory} returned by
* {@link #getMessageBuilderFactory()} to create messages.
* @param useMessageBuilderFactory true if the {@link MessageBuilderFactory} returned
* by {@link #getMessageBuilderFactory()} should be used.
* @since 1.1
*/
public void setUseMessageBuilderFactory(boolean useMessageBuilderFactory) {
@@ -122,9 +100,10 @@ public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport imp
@Override
protected void onInit() {
this.messageListenerContainer.setMessageListener(autoCommitOffset ?
new AutoAcknowledgingChannelForwardingMessageListener()
: new AcknowledgingChannelForwardingMessageListener());
this.messageListenerContainer.setMessageListener(
!AckMode.MANUAL.equals(this.messageListenerContainer.getAckMode())
? new AutoAcknowledgingChannelForwardingMessageListener()
: new AcknowledgingChannelForwardingMessageListener());
if (!this.generateMessageId && !this.generateTimestamp
&& (getMessageBuilderFactory() instanceof DefaultMessageBuilderFactory)) {
setMessageBuilderFactory(new MutableMessageBuilderFactory());
@@ -158,61 +137,46 @@ public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport imp
return getPhase();
}
@SuppressWarnings("rawtypes")
private class AutoAcknowledgingChannelForwardingMessageListener extends AbstractDecodingMessageListener {
@SuppressWarnings("unchecked")
public AutoAcknowledgingChannelForwardingMessageListener() {
super(keyDecoder, payloadDecoder);
}
private class AutoAcknowledgingChannelForwardingMessageListener implements MessageListener<K, V> {
@Override
public void doOnMessage(Object key, Object payload, KafkaMessageMetadata metadata) {
sendMessage(toMessage(key, payload, metadata, null));
public void onMessage(ConsumerRecord<K, V> record) {
sendMessage(toMessage(record, null));
}
}
@SuppressWarnings("rawtypes")
private class AcknowledgingChannelForwardingMessageListener extends AbstractDecodingAcknowledgingMessageListener {
@SuppressWarnings("unchecked")
public AcknowledgingChannelForwardingMessageListener() {
super(keyDecoder, payloadDecoder);
}
private class AcknowledgingChannelForwardingMessageListener implements AcknowledgingMessageListener<K, V> {
@Override
public void doOnMessage(Object key, Object payload, KafkaMessageMetadata metadata,
Acknowledgment acknowledgment) {
sendMessage(toMessage(key, payload, metadata, acknowledgment));
public void onMessage(ConsumerRecord<K, V> record, Acknowledgment acknowledgment) {
sendMessage(toMessage(record, acknowledgment));
}
}
private Message<Object> toMessage(Object key, Object payload, KafkaMessageMetadata metadata,
Acknowledgment acknowledgment) {
private Message<V> toMessage(ConsumerRecord<K, V> record, Acknowledgment acknowledgment) {
KafkaMessageHeaders kafkaMessageHeaders = new KafkaMessageHeaders(generateMessageId, generateTimestamp);
Map<String, Object> rawHeaders = kafkaMessageHeaders.getRawHeaders();
rawHeaders.put(KafkaHeaders.MESSAGE_KEY, key);
rawHeaders.put(KafkaHeaders.TOPIC, metadata.getPartition().getTopic());
rawHeaders.put(KafkaHeaders.PARTITION_ID, metadata.getPartition().getId());
rawHeaders.put(KafkaHeaders.OFFSET, metadata.getOffset());
rawHeaders.put(KafkaHeaders.NEXT_OFFSET, metadata.getNextOffset());
rawHeaders.put(KafkaHeaders.MESSAGE_KEY, record.key());
rawHeaders.put(KafkaHeaders.TOPIC, record.topic());
rawHeaders.put(KafkaHeaders.PARTITION_ID, record.partition());
rawHeaders.put(KafkaHeaders.OFFSET, record.offset());
if (!this.autoCommitOffset) {
if (acknowledgment != null) {
rawHeaders.put(KafkaHeaders.ACKNOWLEDGMENT, acknowledgment);
}
if (this.useMessageBuilderFactory) {
return getMessageBuilderFactory()
.withPayload(payload)
.withPayload(record.value())
.copyHeaders(kafkaMessageHeaders)
.build();
}
else {
return MessageBuilder.createMessage(payload, kafkaMessageHeaders);
return MessageBuilder.createMessage(record.value(), kafkaMessageHeaders);
}
}
@@ -229,4 +193,5 @@ public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport imp
}
}
}

62
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingAcknowledgingMessageListener.java
View File

@@ -1,62 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import org.springframework.integration.kafka.core.KafkaMessage;
import org.springframework.integration.kafka.core.KafkaMessageMetadata;
import org.springframework.integration.kafka.util.MessageUtils;
import kafka.serializer.Decoder;
/**
* Base {@link AcknowledgingMessageListener} implementation that decodes the key and the
* payload using the supplied {@link Decoder}s.
*
* Users of this class must extend it and implement {@code doOnMessage} and must supply
* {@link Decoder} implementations for both the key and the payload.
*
* @author Marius Bogoevici
* @since 1.0.1
*/
public abstract class AbstractDecodingAcknowledgingMessageListener<K, P> implements AcknowledgingMessageListener {
private final Decoder<K> keyDecoder;
private final Decoder<P> payloadDecoder;
public AbstractDecodingAcknowledgingMessageListener(Decoder<K> keyDecoder, Decoder<P> payloadDecoder) {
this.keyDecoder = keyDecoder;
this.payloadDecoder = payloadDecoder;
}
@Override
public final void onMessage(KafkaMessage message, Acknowledgment acknowledgment) {
this.doOnMessage(MessageUtils.decodeKey(message, keyDecoder),
MessageUtils.decodePayload(message, payloadDecoder), message.getMetadata(), acknowledgment);
}
/**
* Process the decoded message
* @param key the message key
* @param payload the message body
* @param metadata the KafkaMessageMetadata
* @param acknowledgment the acknowledgment handle
*/
public abstract void doOnMessage(K key, P payload, KafkaMessageMetadata metadata, Acknowledgment acknowledgment);
}

62
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingMessageListener.java
View File

@@ -1,62 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import kafka.serializer.Decoder;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.integration.kafka.core.KafkaMessage;
import org.springframework.integration.kafka.core.KafkaMessageMetadata;
import org.springframework.integration.kafka.util.MessageUtils;
import org.springframework.util.Assert;
/**
* Base {@link MessageListener} implementation that decodes the key and the payload using the supplied
* {@link Decoder}s.
*
* Users of this class must extend it and implement {@code doOnMessage} and must supply {@link Decoder}
* implementations for both the key and the payload.
*
* @author Marius Bogoevici
*/
public abstract class AbstractDecodingMessageListener<K, P> implements MessageListener {
private Decoder<K> keyDecoder;
private Decoder<P> payloadDecoder;
public AbstractDecodingMessageListener(Decoder<K> keyDecoder, Decoder<P> payloadDecoder) {
this.keyDecoder = keyDecoder;
this.payloadDecoder = payloadDecoder;
}
@Override
public final void onMessage(KafkaMessage message) {
this.doOnMessage(MessageUtils.decodeKey(message, keyDecoder),
MessageUtils.decodePayload(message, payloadDecoder), message.getMetadata());
}
/**
* Process the decoded message
* @param key the message key
* @param payload the message body
* @param metadata the KafkaMessageMetadata
*/
public abstract void doOnMessage(K key, P payload, KafkaMessageMetadata metadata);
}

176
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractOffsetManager.java
View File

@@ -1,176 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import kafka.common.ErrorMapping;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.integration.kafka.core.BrokerAddress;
import org.springframework.integration.kafka.core.Connection;
import org.springframework.integration.kafka.core.ConnectionFactory;
import org.springframework.integration.kafka.core.ConsumerException;
import org.springframework.integration.kafka.core.KafkaConsumerDefaults;
import org.springframework.integration.kafka.core.Partition;
import org.springframework.integration.kafka.core.PartitionNotFoundException;
import org.springframework.integration.kafka.core.Result;
import org.springframework.util.Assert;
/**
* Base implementation for {@link OffsetManager}. Subclasses may customize functionality as necessary.
*
* @author Marius Bogoevici
*/
public abstract class AbstractOffsetManager implements OffsetManager, DisposableBean {
protected final Log log = LogFactory.getLog(this.getClass());
protected String consumerId = KafkaConsumerDefaults.GROUP_ID;
protected long referenceTimestamp = KafkaConsumerDefaults.DEFAULT_OFFSET_RESET;
protected ConnectionFactory connectionFactory;
protected Map<Partition, Long> initialOffsets;
protected Map<Partition, Long> highestUpdatedOffsets = new ConcurrentHashMap<Partition, Long>();
public AbstractOffsetManager(ConnectionFactory connectionFactory) {
this(connectionFactory, new HashMap<Partition, Long>());
}
public AbstractOffsetManager(ConnectionFactory connectionFactory, Map<Partition, Long> initialOffsets) {
Assert.notNull(connectionFactory, "A 'connectionFactory' can't be null");
Assert.notNull(initialOffsets, "An initialOffsets can't be null");
this.connectionFactory = connectionFactory;
this.initialOffsets = new HashMap<Partition, Long>(initialOffsets);
}
public String getConsumerId() {
return this.consumerId;
}
/**
* The identifier of a consumer of Kafka messages. Allows to manage offsets separately by consumer
*
* @param consumerId the consumer ID
*/
public void setConsumerId(String consumerId) {
this.consumerId = consumerId;
}
/**
* A timestamp to be used for resetting initial offsets
*
* @param referenceTimestamp the reset timestamp for initial offsets
*/
public void setReferenceTimestamp(long referenceTimestamp) {
this.referenceTimestamp = referenceTimestamp;
}
@Override
public void destroy() throws Exception {
try {
this.flush();
}
catch (IOException e) {
log.error("Error while flushing the OffsetManager", e);
}
try {
this.close();
}
catch (IOException e) {
log.error("Error while closing the OffsetManager", e);
}
}
/**
* @see OffsetManager#updateOffset(Partition, long)
*/
@Override
public synchronized final void updateOffset(Partition partition, long offset) {
Long highestUpdatedOffset = this.highestUpdatedOffsets.get(partition);
if (highestUpdatedOffset == null || highestUpdatedOffset < offset) {
highestUpdatedOffsets.put(partition, offset);
doUpdateOffset(partition, offset);
}
}
/**
* @see OffsetManager#getOffset(Partition)
*/
@Override
public synchronized final long getOffset(Partition partition) {
Long storedOffset = doGetOffset(partition);
if (storedOffset == null) {
if (this.initialOffsets.containsKey(partition)) {
return this.initialOffsets.get(partition);
}
else {
BrokerAddress leader = this.connectionFactory.getLeader(partition);
if (leader == null) {
throw new PartitionNotFoundException(partition);
}
Connection connection = this.connectionFactory.connect(leader);
Result<Long> offsetResult = connection.fetchInitialOffset(this.referenceTimestamp, partition);
if (offsetResult.getErrors().size() > 0) {
short errorCode = offsetResult.getError(partition);
if (log.isWarnEnabled()) {
log.warn("Error code while retrieving offset for partition " + partition.toString() + " : " + errorCode);
}
throw new ConsumerException(ErrorMapping.exceptionFor(errorCode));
}
if (!offsetResult.getResults().containsKey(partition)) {
throw new IllegalStateException("Result does not contain an expected value");
}
return offsetResult.getResult(partition);
}
}
else {
return storedOffset;
}
}
@Override
public synchronized void resetOffsets(Collection<Partition> partitionsToReset) {
// any information about those offsets is invalid and can be ignored
for (Partition partition : partitionsToReset) {
doRemoveOffset(partition);
this.initialOffsets.remove(partition);
this.highestUpdatedOffsets.remove(partition);
}
}
@Override
public synchronized void deleteOffset(Partition partition) {
doRemoveOffset(partition);
}
protected abstract void doUpdateOffset(Partition partition, long offset);
protected abstract void doRemoveOffset(Partition partition);
protected abstract Long doGetOffset(Partition partition);
}

38
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AcknowledgingMessageListener.java
View File

@@ -1,38 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import org.springframework.integration.kafka.core.KafkaMessage;
/**
* Listener for handling incoming Kafka messages, propagating an acknowledgment handle that recipients
* can invoke when the message has been processed.
*
* @author Marius Bogoevici
* @since 1.0.1
*/
public interface AcknowledgingMessageListener {
/**
* Executes when a Kafka message is received
*
* @param message the Kafka message to be processed
* @param acknowledgment a handle for acknowledging the message processing
*/
void onMessage(KafkaMessage message, Acknowledgment acknowledgment);
}

37
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/Acknowledgment.java
View File

@@ -1,37 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import org.springframework.integration.kafka.core.KafkaMessage;
/**
* Handle for acknowledging the processing of a {@link KafkaMessage}. Recipients can store the reference in
* asynchronous scenarios, but the internal state should be assumed transient (i.e. it cannot be serialized
* and deserialized later)
*
* @author Marius Bogoevici
* @since 1.0.1
*/
public interface Acknowledgment {
/**
* Invoked when the message for which the acknowledgment has been created has been processed.
* Calling this method implies that all the previous messages in the partition have been processed already.
*/
void acknowledge();
}

167
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ConcurrentMessageListenerDispatcher.java
View File

@@ -1,167 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.Executor;
import com.gs.collections.api.block.procedure.Procedure;
import com.gs.collections.api.block.procedure.Procedure2;
import com.gs.collections.api.map.MutableMap;
import com.gs.collections.impl.factory.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.integration.kafka.core.KafkaMessage;
import org.springframework.integration.kafka.core.Partition;
import org.springframework.util.Assert;
/**
* Dispatches {@link KafkaMessage}s to a {@link MessageListener}. Messages may be
* processed concurrently, according to the {@code concurrency} settings, but messages
* from the same partition are being processed in their original order.
*
* @author Marius Bogoevici
* @author Artem Bilan
*/
class ConcurrentMessageListenerDispatcher {
private static final Log log = LogFactory.getLog(ConcurrentMessageListenerDispatcher.class);
private static final StartDelegateProcedure startDelegateProcedure = new StartDelegateProcedure();
private static final StopDelegateProcedure stopDelegateProcedure = new StopDelegateProcedure();
private final Object lifecycleMonitor = new Object();
private final Collection<Partition> partitions;
private final int consumers;
private final Object delegateListener;
private final ErrorHandler errorHandler;
private final OffsetManager offsetManager;
private final int queueSize;
private final Executor taskExecutor;
private volatile boolean running;
private MutableMap<Partition, QueueingMessageListenerInvoker> delegates;
private boolean autoCommitOnError;
public ConcurrentMessageListenerDispatcher(Object delegateListener, ErrorHandler errorHandler,
Collection<Partition> partitions, OffsetManager offsetManager, int consumers, int queueSize,
Executor taskExecutor, boolean autoCommitOnError) {
Assert.isTrue(
delegateListener instanceof MessageListener || delegateListener instanceof AcknowledgingMessageListener,
"Either a " + MessageListener.class.getName() + " or a " + AcknowledgingMessageListener.class.getName()
+ " must be provided");
Assert.notEmpty(partitions, "A set of partitions must be provided");
Assert.isTrue(consumers <= partitions.size(),
"The number of consumers must be smaller or equal to the number of partitions");
Assert.notNull(delegateListener, "A delegate must be provided");
this.delegateListener = delegateListener;
this.errorHandler = errorHandler;
this.partitions = partitions;
this.offsetManager = offsetManager;
this.consumers = Math.min(partitions.size(), consumers);
this.queueSize = queueSize;
this.taskExecutor = taskExecutor;
this.autoCommitOnError = autoCommitOnError;
}
public void start() {
synchronized (lifecycleMonitor) {
if (!this.running) {
initializeAndStartDispatching();
this.running = true;
}
}
}
public void stop(int stopTimeout) {
synchronized (lifecycleMonitor) {
if (this.running) {
this.running = false;
delegates.flip().keyBag().toSet().forEachWith(stopDelegateProcedure, stopTimeout);
}
this.delegates = null;
}
}
public void dispatch(KafkaMessage message) {
if (this.running) {
delegates.get(message.getMetadata().getPartition()).enqueue(message);
}
}
private void initializeAndStartDispatching() {
// allocate delegate instances index them
List<QueueingMessageListenerInvoker> delegateList = new ArrayList<>(consumers);
for (int i = 0; i < consumers; i++) {
QueueingMessageListenerInvoker queueingMessageListenerInvoker = new QueueingMessageListenerInvoker(
queueSize, offsetManager, delegateListener, errorHandler, taskExecutor, autoCommitOnError);
delegateList.add(queueingMessageListenerInvoker);
}
// evenly distribute partitions across delegates
int i = 0;
delegates = Maps.mutable.of();
for (Partition partition : partitions) {
delegates.put(partition, delegateList.get((i++) % consumers));
}
// start dispatchers
delegates.flip().keyBag().toSet().forEach(startDelegateProcedure);
}
@SuppressWarnings("serial")
private static class StopDelegateProcedure implements Procedure2<QueueingMessageListenerInvoker, Integer> {
@Override
public void value(QueueingMessageListenerInvoker delegate, Integer stopTimeout) {
try {
delegate.stop(stopTimeout);
}
catch (Exception e) {
// ignore the exception, but log it
if (log.isInfoEnabled()) {
log.info("Exception thrown while stopping dispatcher:", e);
}
}
}
}
@SuppressWarnings("serial")
private static class StartDelegateProcedure implements Procedure<QueueingMessageListenerInvoker> {
@Override
public void value(QueueingMessageListenerInvoker delegate) {
delegate.start();
}
}
}

52
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/DefaultAcknowledgment.java
View File

@@ -1,52 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import org.springframework.integration.kafka.core.KafkaMessage;
import org.springframework.integration.kafka.core.Partition;
/**
* Default implementation for an {@link Acknowledgment} that defers to an underlying
* {@link OffsetManager}.
*
* @author Marius Bogoevici
* @since 1.0.1
*/
public class DefaultAcknowledgment implements Acknowledgment {
private final OffsetManager offsetManager;
private final Partition partition;
private final Long offset;
public DefaultAcknowledgment(OffsetManager offsetManager, Partition partition, Long offset) {
this.offsetManager = offsetManager;
this.partition = partition;
this.offset = offset;
}
public DefaultAcknowledgment(OffsetManager offsetManager, KafkaMessage message) {
this(offsetManager, message.getMetadata().getPartition(), message.getMetadata().getNextOffset());
}
@Override
public void acknowledge() {
offsetManager.updateOffset(partition, offset);
}
}

31
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ErrorHandler.java
View File

@@ -1,31 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import org.springframework.integration.kafka.core.KafkaMessage;
/**
* Handles errors thrown during the execution of a {@link MessageListener}
*
* @author Marius Bogoevici
*/
public interface ErrorHandler {
void handle(Exception thrownException, KafkaMessage message);
}

655
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaMessageListenerContainer.java
View File

@@ -1,655 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import com.gs.collections.api.RichIterable;
import com.gs.collections.api.block.function.Function;
import com.gs.collections.api.block.predicate.Predicate;
import com.gs.collections.api.list.ImmutableList;
import com.gs.collections.api.list.MutableList;
import com.gs.collections.api.multimap.list.ImmutableListMultimap;
import com.gs.collections.api.multimap.set.MutableSetMultimap;
import com.gs.collections.api.partition.PartitionIterable;
import com.gs.collections.api.set.MutableSet;
import com.gs.collections.api.tuple.Pair;
import com.gs.collections.impl.block.factory.Functions;
import com.gs.collections.impl.block.function.checked.CheckedFunction;
import com.gs.collections.impl.factory.Lists;
import com.gs.collections.impl.factory.Sets;
import com.gs.collections.impl.list.mutable.FastList;
import com.gs.collections.impl.map.mutable.UnifiedMap;
import com.gs.collections.impl.utility.ArrayIterate;
import com.gs.collections.impl.utility.Iterate;
import kafka.common.ErrorMapping;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.context.SmartLifecycle;
import org.springframework.core.task.SimpleAsyncTaskExecutor;
import org.springframework.integration.kafka.core.BrokerAddress;
import org.springframework.integration.kafka.core.ConnectionFactory;
import org.springframework.integration.kafka.core.ConsumerException;
import org.springframework.integration.kafka.core.FetchRequest;
import org.springframework.integration.kafka.core.KafkaConsumerDefaults;
import org.springframework.integration.kafka.core.KafkaMessage;
import org.springframework.integration.kafka.core.KafkaMessageBatch;
import org.springframework.integration.kafka.core.KafkaTemplate;
import org.springframework.integration.kafka.core.Partition;
import org.springframework.integration.kafka.core.Result;
import org.springframework.scheduling.SchedulingAwareRunnable;
import org.springframework.util.Assert;
/**
* @author Marius Bogoevici
*/
public class KafkaMessageListenerContainer implements SmartLifecycle {
public static final int DEFAULT_WAIT_FOR_LEADER_REFRESH_RETRY = 5000;
private static final int DEFAULT_STOP_TIMEOUT = 1000;
private static final Log log = LogFactory.getLog(KafkaMessageListenerContainer.class);
private final GetOffsetForPartitionFunction getOffset = new GetOffsetForPartitionFunction();
private final PartitionToLeaderFunction getLeader = new PartitionToLeaderFunction();
private final Function<Partition, Partition> passThru = Functions.getPassThru();
private final Object lifecycleMonitor = new Object();
private final KafkaTemplate kafkaTemplate;
private final String[] topics;
private Partition[] partitions;
public boolean autoStartup = true;
private Executor fetchTaskExecutor;
private Executor adminTaskExecutor;
private Executor dispatcherTaskExecutor;
private int concurrency = 1;
private volatile boolean running = false;
private int maxFetch = KafkaConsumerDefaults.FETCH_SIZE_INT;
private int queueSize = 1024;
private int stopTimeout = DEFAULT_STOP_TIMEOUT;
private Object messageListener;
private ErrorHandler errorHandler = new LoggingErrorHandler();
private volatile OffsetManager offsetManager;
private ConcurrentMap<Partition, Long> fetchOffsets;
private ConcurrentMessageListenerDispatcher messageDispatcher;
private final ConcurrentMap<BrokerAddress, FetchTask> fetchTasksByBroker = new ConcurrentHashMap<>();
private boolean autoCommitOnError = false;
public KafkaMessageListenerContainer(ConnectionFactory connectionFactory, Partition... partitions) {
Assert.notNull(connectionFactory, "A connection factory must be supplied");
Assert.notEmpty(partitions, "A list of partitions must be provided");
Assert.noNullElements(partitions, "The list of partitions cannot contain null elements");
this.kafkaTemplate = new KafkaTemplate(connectionFactory);
this.partitions = partitions;
this.topics = null;
}
public KafkaMessageListenerContainer(final ConnectionFactory connectionFactory, String... topics) {
Assert.notNull(connectionFactory, "A connection factory must be supplied");
Assert.notNull(topics, "A list of topics must be provided");
Assert.noNullElements(topics, "The list of topics cannot contain null elements");
this.kafkaTemplate = new KafkaTemplate(connectionFactory);
this.topics = topics;
}
public OffsetManager getOffsetManager() {
return offsetManager;
}
public void setOffsetManager(OffsetManager offsetManager) {
this.offsetManager = offsetManager;
}
public Object getMessageListener() {
return messageListener;
}
public void setMessageListener(Object messageListener) {
Assert.isTrue(
messageListener instanceof MessageListener || messageListener instanceof AcknowledgingMessageListener,
"Either a " + MessageListener.class.getName() + " or a " + AcknowledgingMessageListener.class.getName()
+ " must be provided");
this.messageListener = messageListener;
}
public ErrorHandler getErrorHandler() {
return errorHandler;
}
public void setErrorHandler(ErrorHandler errorHandler) {
this.errorHandler = errorHandler;
}
public int getConcurrency() {
return concurrency;
}
/**
* The maximum number of concurrent {@link MessageListener}s running. Messages from
* within the same partition will be processed sequentially.
* @param concurrency the concurrency maximum number
*/
public void setConcurrency(int concurrency) {
this.concurrency = concurrency;
}
/**
* The timeout for waiting for each concurrent {@link MessageListener} to finish on
* stopping.
* @param stopTimeout timeout in milliseconds
* @since 1.1
*/
public void setStopTimeout(int stopTimeout) {
this.stopTimeout = stopTimeout;
}
public int getStopTimeout() {
return stopTimeout;
}
public Executor getFetchTaskExecutor() {
return fetchTaskExecutor;
}
/**
* The task executor for fetch operations.
* @param fetchTaskExecutor the Executor for fetch operations
*/
public void setFetchTaskExecutor(Executor fetchTaskExecutor) {
this.fetchTaskExecutor = fetchTaskExecutor;
}
public Executor getAdminTaskExecutor() {
return adminTaskExecutor;
}
/**
* The task executor for leader, offset, and partition reassignment updates.
* @param adminTaskExecutor the task executor for leader, offset and partition reassignment updates
*/
public void setAdminTaskExecutor(Executor adminTaskExecutor) {
this.adminTaskExecutor = adminTaskExecutor;
}
/**
* The task executor for invoking the MessageListener
* @param dispatcherTaskExecutor the task executor for invoking the MessageListener
*/
public void setDispatcherTaskExecutor(Executor dispatcherTaskExecutor) {
this.dispatcherTaskExecutor = dispatcherTaskExecutor;
}
/**
* @return the maximum amount of data (in bytes) that pollers will fetch in one round
*/
public int getMaxFetch() {
return maxFetch;
}
public int getQueueSize() {
return queueSize;
}
/**
* The maximum number of messages that are buffered by each concurrent
* {@link MessageListener} runner. Increasing the value may increase throughput, but
* also increases the memory consumption. Must be a positive number and a power of 2.
* @param queueSize the queue size
*/
public void setQueueSize(int queueSize) {
Assert.isTrue(queueSize > 0 && Integer.bitCount(queueSize) == 1,
"'queueSize' must be a positive number and a power of 2");
this.queueSize = queueSize;
}
public void setMaxFetch(int maxFetch) {
this.maxFetch = maxFetch;
}
/**
* Whether offsets should be auto acknowledged even when exceptions are thrown during processing. This setting
* is effective only in auto acknowledged mode. When set to true, all received messages will be acknowledged,
* and when set to false only the offset of the last successfully processed message is persisted, even if the
* component will try to continue processing incoming messages. In the latter case, it is possible that
* a successful message will commit an offset after a series of failures, so the component should rely on
* the `errorHandler` to capture failures.
* @param autoCommitOnError false if offsets should be committed only for successful messages
* @since 1.3
*/
public void setAutoCommitOnError(boolean autoCommitOnError) {
this.autoCommitOnError = autoCommitOnError;
}
public boolean isAutoCommitOnError() {
return autoCommitOnError;
}
@Override
public boolean isAutoStartup() {
return autoStartup;
}
public void setAutoStartup(boolean autoStartup) {
this.autoStartup = autoStartup;
}
@Override
public void stop(Runnable callback) {
synchronized (lifecycleMonitor) {
if (running) {
this.running = false;
try {
this.offsetManager.flush();
}
catch (IOException e) {
log.error("Error while flushing:", e);
}
this.messageDispatcher.stop(stopTimeout);
}
}
if (callback != null) {
callback.run();
}
}
@Override
public void start() {
synchronized (lifecycleMonitor) {
if (!running) {
if (partitions == null) {
partitions = getPartitionsForTopics(kafkaTemplate.getConnectionFactory(), topics);
}
this.running = true;
if (this.offsetManager == null) {
this.offsetManager = new MetadataStoreOffsetManager(kafkaTemplate.getConnectionFactory());
}
// initialize the fetch offset table - defer to OffsetManager for retrieving them
ImmutableList<Partition> partitionsAsList = Lists.immutable.with(partitions);
this.fetchOffsets = new ConcurrentHashMap<Partition, Long>(partitionsAsList.toMap(passThru, getOffset));
this.messageDispatcher = new ConcurrentMessageListenerDispatcher(messageListener, errorHandler,
Arrays.asList(partitions), offsetManager, concurrency, queueSize, dispatcherTaskExecutor,
autoCommitOnError);
this.messageDispatcher.start();
fetchTasksByBroker.clear();
ImmutableListMultimap<BrokerAddress, Partition> partitionsByLeader = partitionsAsList
.groupBy(getLeader);
if (fetchTaskExecutor == null) {
fetchTaskExecutor = new SimpleAsyncTaskExecutor("kafka-fetch-");
}
if (adminTaskExecutor == null) {
adminTaskExecutor = Executors.newSingleThreadExecutor();
}
for (Pair<BrokerAddress, RichIterable<Partition>> entry : partitionsByLeader.keyMultiValuePairsView()) {
FetchTask fetchTask = new FetchTask(entry.getOne(), entry.getTwo());
fetchTaskExecutor.execute(fetchTask);
fetchTasksByBroker.put(entry.getOne(), fetchTask);
}
}
}
}
@Override
public void stop() {
this.stop(null);
}
@Override
public boolean isRunning() {
return this.running;
}
@Override
public int getPhase() {
return 0;
}
private static Partition[] getPartitionsForTopics(final ConnectionFactory connectionFactory, String[] topics) {
MutableList<Partition> partitionList =
ArrayIterate.flatCollect(topics, new GetPartitionsForTopic(connectionFactory));
return partitionList.toArray(new Partition[partitionList.size()]);
}
/**
* Fetches data from Kafka for a group of partitions, located on the same broker.
*/
public class FetchTask implements SchedulingAwareRunnable {
private final BrokerAddress brokerAddress;
private final MutableSet<Partition> listenedPartitions = Sets.mutable.<Partition>of().asSynchronized();
private volatile boolean active;
private final PartitionToFetchRequestFunction partitionToFetchRequestFunction =
new PartitionToFetchRequestFunction();
private final IsLeaderErrorPredicate isLeaderPredicate = new IsLeaderErrorPredicate();
private final IsOffsetOutOfRangePredicate offsetOutOfRangePredicate = new IsOffsetOutOfRangePredicate();
public FetchTask(BrokerAddress brokerAddress, RichIterable<Partition> initialPartitions) {
this.brokerAddress = brokerAddress;
this.active = true;
this.listenedPartitions.addAll(initialPartitions.toSet());
}
@Override
public boolean isLongLived() {
return true;
}
public boolean addListenedPartitionsIfActive(Iterable<Partition> partitions) {
synchronized (listenedPartitions) {
if (active) {
listenedPartitions.addAllIterable(partitions);
}
return active;
}
}
@Override
public void run() {
try {
while (active && isRunning()) {
synchronized (listenedPartitions) {
try {
if (!listenedPartitions.isEmpty()) {
Result<KafkaMessageBatch> result = fetchAvailableData();
handleSuccessful(result);
if (result.getErrors().size() > 0) {
handleErrors(result);
}
}
else {
active = false;
}
}
catch (ConsumerException e) {
active = false;
// the connection is broken, terminate the task
kafkaTemplate.getConnectionFactory().disconnect(brokerAddress);
resetLeaders(listenedPartitions.toImmutable());
}
}
}
}
finally {
active = false;
synchronized (fetchTasksByBroker) {
if (fetchTasksByBroker.get(brokerAddress) == this) {
fetchTasksByBroker.remove(brokerAddress);
}
}
}
}
private Result<KafkaMessageBatch> fetchAvailableData() {
return kafkaTemplate.receive(listenedPartitions.collect(partitionToFetchRequestFunction));
}
private void handleSuccessful(Result<KafkaMessageBatch> result) {
Iterable<KafkaMessageBatch> batches = result.getResults().values();
for (KafkaMessageBatch batch : batches) {
if (!batch.getMessages().isEmpty()) {
long highestFetchedOffset = 0;
for (KafkaMessage kafkaMessage : batch.getMessages()) {
// fetch operations may return entire blocks of compressed messages,
// which may have lower offsets than the ones requested
// thus a batch may contain messages that have been processed already
if (kafkaMessage.getMetadata().getOffset() >= fetchOffsets.get(batch.getPartition())) {
messageDispatcher.dispatch(kafkaMessage);
}
highestFetchedOffset = Math.max(highestFetchedOffset, kafkaMessage.getMetadata().getNextOffset());
}
fetchOffsets.replace(batch.getPartition(), highestFetchedOffset);
}
}
}
private void handleErrors(Result<KafkaMessageBatch> result) {
Map<Partition, Short> errors = result.getErrors();
PartitionIterable<Map.Entry<Partition, Short>> splitByLeaderError =
Iterate.partition(errors.entrySet(), isLeaderPredicate);
RichIterable<Partition> partitionsWithLeaderErrors = splitByLeaderError.getSelected()
.collect(Functions.<Partition>getKeyFunction());
resetLeaders(partitionsWithLeaderErrors);
PartitionIterable<Map.Entry<Partition, Short>> splitByOffsetError =
splitByLeaderError.getRejected().partition(offsetOutOfRangePredicate);
RichIterable<Partition> partitionsWithWrongOffsets =
splitByOffsetError.getSelected().collect(Functions.<Partition>getKeyFunction());
resetOffsets(partitionsWithWrongOffsets.toSet());
// it's not a leader issue, remove everything else
RichIterable<Partition> remainingPartitionsWithErrors
= splitByOffsetError.getRejected().collect(Functions.<Partition>getKeyFunction());
listenedPartitions.removeAllIterable(remainingPartitionsWithErrors);
}
private void resetLeaders(final Iterable<Partition> partitionsToReset) {
listenedPartitions.removeAllIterable(partitionsToReset);
adminTaskExecutor.execute(new UpdateLeadersTask(partitionsToReset));
}
private void resetOffsets(final Collection<Partition> partitionsToResetOffsets) {
listenedPartitions.removeAllIterable(partitionsToResetOffsets);
adminTaskExecutor.execute(new UpdateOffsetsTask(partitionsToResetOffsets));
}
private class UpdateLeadersTask implements SchedulingAwareRunnable {
private final Iterable<Partition> partitionsToReset;
public UpdateLeadersTask(Iterable<Partition> partitionsToReset) {
this.partitionsToReset = partitionsToReset;
}
@Override
public boolean isLongLived() {
return true;
}
@Override
public void run() {
// fetch can complete successfully or unsuccessfully
boolean fetchCompleted = false;
while (!fetchCompleted && isRunning()) {
try {
FastList<Partition> partitionsAsList = FastList.newList(partitionsToReset);
FastList<String> topics = partitionsAsList.collect(new PartitionToTopicFunction()).distinct();
kafkaTemplate.getConnectionFactory().refreshMetadata(topics);
MutableSetMultimap<BrokerAddress, Partition> partitionsByBroker = UnifiedMap
.newMap(kafkaTemplate.getConnectionFactory().getLeaders(partitionsToReset)).flip();
for (Pair<BrokerAddress, RichIterable<Partition>> pair : partitionsByBroker
.keyMultiValuePairsView()) {
synchronized (fetchTasksByBroker) {
boolean addedSuccessfully = false;
FetchTask fetchTask = fetchTasksByBroker.get(pair.getOne());
if (fetchTask != null) {
addedSuccessfully = fetchTask.addListenedPartitionsIfActive(pair.getTwo());
}
if (!addedSuccessfully) {
fetchTask = new FetchTask(pair.getOne(), pair.getTwo());
fetchTaskExecutor.execute(fetchTask);
fetchTasksByBroker.put(pair.getOne(), fetchTask);
}
}
}
fetchCompleted = true;
}
catch (Exception e) {
if (isRunning()) {
try {
Thread.sleep(DEFAULT_WAIT_FOR_LEADER_REFRESH_RETRY);
}
catch (InterruptedException e1) {
Thread.currentThread().interrupt();
log.error("Interrupted after refresh leaders failure for: " + Iterate
.makeString(partitionsToReset, ","));
fetchCompleted = true;
}
}
}
}
}
}
private class UpdateOffsetsTask implements Runnable {
private final Collection<Partition> partitionsToResetOffsets;
public UpdateOffsetsTask(Collection<Partition> partitionsToResetOffsets) {
this.partitionsToResetOffsets = partitionsToResetOffsets;
}
@Override
public void run() {
offsetManager.resetOffsets(partitionsToResetOffsets);
for (Partition partition : partitionsToResetOffsets) {
fetchOffsets.replace(partition, offsetManager.getOffset(partition));
}
synchronized (fetchTasksByBroker) {
boolean addedSuccessfully = false;
FetchTask fetchTask = fetchTasksByBroker.get(brokerAddress);
if (fetchTask != null) {
addedSuccessfully = fetchTask.addListenedPartitionsIfActive(partitionsToResetOffsets);
}
if (!addedSuccessfully) {
fetchTask = new FetchTask(brokerAddress, Sets.immutable.ofAll(partitionsToResetOffsets));
fetchTaskExecutor.execute(fetchTask);
fetchTasksByBroker.put(brokerAddress, fetchTask);
}
}
}
}
@SuppressWarnings("serial")
private class IsLeaderErrorPredicate implements Predicate<Map.Entry<Partition, Short>> {
@Override
public boolean accept(Map.Entry<Partition, Short> each) {
return each.getValue() == ErrorMapping.NotLeaderForPartitionCode() || each.getValue() == ErrorMapping
.UnknownTopicOrPartitionCode();
}
}
@SuppressWarnings("serial")
private class IsOffsetOutOfRangePredicate implements Predicate<Map.Entry<Partition, Short>> {
@Override
public boolean accept(Map.Entry<Partition, Short> each) {
return each.getValue() == ErrorMapping.OffsetOutOfRangeCode();
}
}
}
@SuppressWarnings("serial")
class GetOffsetForPartitionFunction extends CheckedFunction<Partition, Long> {
@Override
public Long safeValueOf(Partition object) throws Exception {
try {
return offsetManager.getOffset(object);
}
catch (Exception e) {
log.error(e);
throw e;
}
}
}
@SuppressWarnings("serial")
private class PartitionToLeaderFunction implements Function<Partition, BrokerAddress> {
@Override
public BrokerAddress valueOf(Partition partition) {
return kafkaTemplate.getConnectionFactory().getLeader(partition);
}
}
@SuppressWarnings("serial")
private class PartitionToFetchRequestFunction implements Function<Partition, FetchRequest> {
@Override
public FetchRequest valueOf(Partition partition) {
return new FetchRequest(partition, fetchOffsets.get(partition), maxFetch);
}
}
@SuppressWarnings("serial")
static class GetPartitionsForTopic extends CheckedFunction<String, Iterable<Partition>> {
private final ConnectionFactory connectionFactory;
public GetPartitionsForTopic(ConnectionFactory connectionFactory) {
this.connectionFactory = connectionFactory;
}
@Override
public Iterable<Partition> safeValueOf(String topic) throws Exception {
return connectionFactory.getPartitions(topic);
}
}
@SuppressWarnings("serial")
private class PartitionToTopicFunction implements Function<Partition, String> {
@Override
public String valueOf(Partition object) {
return object.getTopic();
}
}
}

231
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaNativeOffsetManager.java
View File

@@ -1,231 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.I0Itec.zkclient.ZkClient;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.integration.kafka.core.BrokerAddress;
import org.springframework.integration.kafka.core.Configuration;
import org.springframework.integration.kafka.core.Connection;
import org.springframework.integration.kafka.core.ConnectionFactory;
import org.springframework.integration.kafka.core.ConsumerException;
import org.springframework.integration.kafka.core.DefaultConnectionFactory;
import org.springframework.integration.kafka.core.KafkaConsumerDefaults;
import org.springframework.integration.kafka.core.Partition;
import org.springframework.integration.kafka.core.PartitionNotFoundException;
import org.springframework.integration.kafka.core.Result;
import org.springframework.integration.kafka.support.ZookeeperConnect;
import org.springframework.retry.RetryCallback;
import org.springframework.retry.RetryContext;
import org.springframework.retry.backoff.ExponentialBackOffPolicy;
import org.springframework.retry.listener.RetryListenerSupport;
import org.springframework.retry.policy.SimpleRetryPolicy;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.util.Assert;
import com.gs.collections.impl.factory.Maps;
import kafka.client.ClientUtils$;
import kafka.common.ErrorMapping;
import kafka.common.OffsetAndMetadata;
import kafka.network.BlockingChannel;
import kafka.utils.ZKStringSerializer$;
/**
* Implementation of an {@link OffsetManager} that uses kafka native 'topic' offset storage.
*
* @author Chris Lemper
* @author Marius Bogoevici
* @since 1.3
*/
public class KafkaNativeOffsetManager extends AbstractOffsetManager implements InitializingBean {
private static final String PARTITION_ATTRIBUTE = "partition";
private final Map<Partition, BrokerAddress> offsetManagerBrokerAddressCache = new ConcurrentHashMap<>();
private final ZkClient zkClient;
private RetryTemplate retryTemplate;
/**
* @param connectionFactory a Kafka connection factory
* @param zookeeperConnect the zookeeper connection information
*/
public KafkaNativeOffsetManager(ConnectionFactory connectionFactory, ZookeeperConnect zookeeperConnect) {
this(connectionFactory, zookeeperConnect, Collections.<Partition, Long>emptyMap());
}
/**
* @param connectionFactory a Kafka connection factory
* @param zookeeperConnect zookeeper connection for retrieving
* @param initialOffsets a map of partitions to initial offsets
*/
public KafkaNativeOffsetManager(ConnectionFactory connectionFactory, ZookeeperConnect zookeeperConnect,
Map<Partition, Long> initialOffsets) {
super(connectionFactory, initialOffsets);
Assert.notNull(zookeeperConnect, "'zookeeperConnect' must not be null.");
this.zkClient = new ZkClient(zookeeperConnect.getZkConnect(),
Integer.parseInt(zookeeperConnect.getZkSessionTimeout()),
Integer.parseInt(zookeeperConnect.getZkConnectionTimeout()),
ZKStringSerializer$.MODULE$);
}
public void setRetryTemplate(RetryTemplate retryTemplate) {
this.retryTemplate = retryTemplate;
}
@Override
public void afterPropertiesSet() throws Exception {
if (this.retryTemplate == null) {
this.retryTemplate = new RetryTemplate();
this.retryTemplate.registerListener(new ResetOffsetManagerBrokerAddressRetryListener());
final SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy();
retryPolicy.setMaxAttempts(5);
this.retryTemplate.setRetryPolicy(retryPolicy);
final ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
backOffPolicy.setInitialInterval(125L);
backOffPolicy.setMaxInterval(5000L);
backOffPolicy.setMultiplier(2);
this.retryTemplate.setBackOffPolicy(backOffPolicy);
}
}
@Override
protected Long doGetOffset(final Partition partition) {
final Long offset = this.retryTemplate.execute(new RetryCallback<Long, RuntimeException>() {
@Override
public Long doWithRetry(RetryContext context) throws RuntimeException {
context.setAttribute(PARTITION_ATTRIBUTE, partition);
Result<Long> result = getOffsetManagerConnection(partition).fetchStoredOffsetsForConsumer(
getConsumerId(), partition);
checkResultForErrors(result, partition);
return result.getResult(partition);
}
@Override
public String toString() {
return String.format("fetchStoredOffsetsForConsumer(%s, %s)", getConsumerId(), partition);
}
});
if (offset != null && offset < 0) {
return null;
}
return offset;
}
@Override
protected void doUpdateOffset(final Partition partition, final long offset) {
this.retryTemplate.execute(new RetryCallback<Void, RuntimeException>() {
@Override
public Void doWithRetry(RetryContext context) throws RuntimeException {
context.setAttribute(PARTITION_ATTRIBUTE, partition);
Result<Void> result = getOffsetManagerConnection(partition).commitOffsetsForConsumer(
getConsumerId(), Maps.immutable.of(partition, offset).castToMap());
checkResultForErrors(result, partition);
return null;
}
@Override
public String toString() {
return String.format("commitOffsetsForConsumer(%s, %s, %s)", getConsumerId(), partition, offset);
}
});
}
@Override
protected void doRemoveOffset(Partition partition) {
doUpdateOffset(partition, OffsetAndMetadata.InvalidOffset());
}
@Override
public void close() throws IOException {
this.zkClient.close();
}
@Override
public void flush() throws IOException {
// this function left intentionally blank
}
private BrokerAddress getOffsetManagerBrokerAddress(final Partition partition) {
BrokerAddress brokerAddress = this.offsetManagerBrokerAddressCache.get(partition);
if (brokerAddress == null) {
int socketTimeoutMs = KafkaConsumerDefaults.SOCKET_TIMEOUT_INT;
int retryBackOffMs = KafkaConsumerDefaults.BACKOFF_INCREMENT_INT;
if (this.connectionFactory instanceof DefaultConnectionFactory) {
Configuration configuration = ((DefaultConnectionFactory) connectionFactory).getConfiguration();
socketTimeoutMs = configuration.getSocketTimeout();
retryBackOffMs = configuration.getBackOff();
}
final BlockingChannel channel = ClientUtils$.MODULE$.channelToOffsetManager(
getConsumerId(), zkClient, socketTimeoutMs, retryBackOffMs);
brokerAddress = new BrokerAddress(channel.host(), channel.port());
if (log.isDebugEnabled()) {
log.debug(String.format("Offset manager for [%s] is at [%s].", partition,
brokerAddress));
}
this.offsetManagerBrokerAddressCache.put(partition, brokerAddress);
channel.disconnect();
}
return brokerAddress;
}
private Connection getOffsetManagerConnection(final Partition partition) {
return this.connectionFactory.connect(getOffsetManagerBrokerAddress(partition));
}
private void checkResultForErrors(final Result<?> result, final Partition partition) {
if (result.getErrors().containsKey(partition)) {
final short errorCode = result.getError(partition);
if (errorCode == ErrorMapping.UnknownTopicOrPartitionCode()) {
throw new PartitionNotFoundException(partition);
}
else if (errorCode != ErrorMapping.NoError()) {
throw new ConsumerException(ErrorMapping.exceptionFor(errorCode));
}
}
}
private class ResetOffsetManagerBrokerAddressRetryListener extends RetryListenerSupport {
@Override
@SuppressWarnings("unchecked")
public <T, E extends Throwable> void onError(RetryContext context, RetryCallback<T, E> callback, Throwable t) {
if (log.isWarnEnabled()) {
log.warn("Retrying kafka operation [" + callback + "] due to [" + t + "]", t);
}
Partition partition = (Partition) context.getAttribute(PARTITION_ATTRIBUTE);
if (partition != null) {
offsetManagerBrokerAddressCache.remove(partition);
}
}
}
}

506
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaTopicOffsetManager.java
View File

@@ -1,506 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import kafka.admin.AdminUtils$;
import kafka.api.OffsetRequest;
import kafka.common.ErrorMapping$;
import kafka.common.TopicExistsException;
import kafka.serializer.Decoder;
import kafka.utils.ZKStringSerializer$;
import org.I0Itec.zkclient.ZkClient;
import org.I0Itec.zkclient.exception.ZkInterruptedException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.Serializer;
import org.springframework.beans.factory.BeanInitializationException;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.integration.kafka.core.BrokerAddress;
import org.springframework.integration.kafka.core.DefaultConnectionFactory;
import org.springframework.integration.kafka.core.FetchRequest;
import org.springframework.integration.kafka.core.KafkaMessage;
import org.springframework.integration.kafka.core.KafkaMessageBatch;
import org.springframework.integration.kafka.core.KafkaTemplate;
import org.springframework.integration.kafka.core.Partition;
import org.springframework.integration.kafka.core.Result;
import org.springframework.integration.kafka.core.TopicNotFoundException;
import org.springframework.integration.kafka.core.ZookeeperConfiguration;
import org.springframework.integration.kafka.support.ProducerFactoryBean;
import org.springframework.integration.kafka.support.ProducerMetadata;
import org.springframework.integration.kafka.support.ZookeeperConnect;
import org.springframework.integration.kafka.util.LoggingUtils;
import org.springframework.integration.kafka.util.MessageUtils;
import org.springframework.retry.RetryCallback;
import org.springframework.retry.RetryContext;
import org.springframework.retry.backoff.ExponentialBackOffPolicy;
import org.springframework.retry.policy.SimpleRetryPolicy;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.util.Assert;
import org.springframework.util.ObjectUtils;
/**
* Implementation of an {@link OffsetManager} that uses a Kafka topic as the underlying support.
* For its proper functioning, the Kafka server(s) must set {@code log.cleaner.enable=true}. It relies on the property
* {@code cleanup.policy=compact} to be set on the target topic, and if the topic is not found,
* it will create a topic with the appropriate settings.
*
* @author Marius Bogoevici
* @author Artem Bilan
*/
public class KafkaTopicOffsetManager extends AbstractOffsetManager implements InitializingBean {
private static final KeySerializerDecoder KEY_CODEC = new KeySerializerDecoder();
private static final LongSerializerDecoder VALUE_CODEC = new LongSerializerDecoder();
public static final String CLEANUP_POLICY = "cleanup.policy";
public static final String CLEANUP_POLICY_COMPACT = "compact";
public static final String DELETE_RETENTION = "delete.retention.ms";
public static final String SEGMENT_BYTES = "segment.bytes";
private final ZookeeperConnect zookeeperConnect;
private final String topic;
private final KafkaTemplate kafkaTemplate;
private final ConcurrentMap<Partition, Long> data = new ConcurrentHashMap<Partition, Long>();
private ProducerMetadata.CompressionType compressionType = ProducerMetadata.CompressionType.none;
private Producer<Key, Long> producer;
private int maxSize = 10 * 1024;
private int segmentSize = 25 * 1024;
private int retentionTime = 60000;
private int replicationFactor;
private int batchBytes = 200;
private int requiredAcks = 1;
private int maxQueueBufferingTime;
public KafkaTopicOffsetManager(ZookeeperConnect zookeeperConnect, String topic) {
this(zookeeperConnect, topic, new HashMap<Partition, Long>());
}
public KafkaTopicOffsetManager(ZookeeperConnect zookeeperConnect, String topic,
Map<Partition, Long> initialOffsets) {
super(new DefaultConnectionFactory(new ZookeeperConfiguration(zookeeperConnect)), initialOffsets);
Assert.notNull(zookeeperConnect);
this.zookeeperConnect = zookeeperConnect;
this.kafkaTemplate = new KafkaTemplate(connectionFactory);
this.topic = topic;
}
/**
* Sets the maximum size of a fetch request, allowing to tune the initialization process.
*
* @param maxSize the maximum amount of data to be brought on a fetch
*/
public void setMaxSize(int maxSize) {
this.maxSize = maxSize;
}
/**
* The compression type for writing to the offset topic
*
* @param compressionType the compression type
*/
public void setCompressionCodec(ProducerMetadata.CompressionType compressionType) {
this.compressionType = compressionType;
}
/**
* For how long will producers buffer data before writing to the topic
*
* @param maxQueueBufferingTime the maximum buffering window (in milliseconds)
*/
public void setMaxQueueBufferingTime(int maxQueueBufferingTime) {
this.maxQueueBufferingTime = maxQueueBufferingTime;
}
/**
* The size of a segment in the offset topic
*
* @param segmentSize the segment size of an offset topic
*/
public void setSegmentSize(int segmentSize) {
this.segmentSize = segmentSize;
}
/**
* How long are dead records retained in the offset topic
*
* @param retentionTime the retention time for dead records (in seconds)
*/
public void setRetentionTime(int retentionTime) {
this.retentionTime = retentionTime;
}
/**
* The replication factor of the offset topic
*
* @param replicationFactor the replication factor
*/
public void setReplicationFactor(int replicationFactor) {
this.replicationFactor = replicationFactor;
}
/**
* The maximum batch size in bytes for offset writes
*
* @param batchBytes maximum batching window
*/
public void setBatchBytes(int batchBytes) {
this.batchBytes = batchBytes;
}
/**
* The number of required acks on write operations
*
* @param requiredAcks the number of required acks
*/
public void setRequiredAcks(int requiredAcks) {
this.requiredAcks = requiredAcks;
}
@Override
public void afterPropertiesSet() throws Exception {
((DefaultConnectionFactory) this.connectionFactory).afterPropertiesSet();
ZkClient zkClient = new ZkClient(this.zookeeperConnect.getZkConnect(),
Integer.parseInt(this.zookeeperConnect.getZkSessionTimeout()),
Integer.parseInt(this.zookeeperConnect.getZkConnectionTimeout()),
ZKStringSerializer$.MODULE$);
try {
createCompactedTopicIfNotFound(zkClient);
validateOffsetTopic(zkClient);
Partition offsetPartition = new Partition(this.topic, 0);
BrokerAddress offsetPartitionLeader = this.connectionFactory.getLeader(offsetPartition);
readOffsetData(offsetPartition, offsetPartitionLeader);
initializeProducer(offsetPartitionLeader);
}
finally {
try {
zkClient.close();
}
catch (ZkInterruptedException e) {
log.error("Error while closing Zookeeper client", e);
}
}
}
@Override
protected void doUpdateOffset(Partition partition, long offset) {
this.data.put(partition, offset);
this.producer.send(new ProducerRecord<>(this.topic, new Key(this.consumerId, partition), offset));
}
@Override
protected void doRemoveOffset(Partition partition) {
this.data.remove(partition);
this.producer.send(new ProducerRecord<Key, Long>(this.topic, new Key(this.consumerId, partition), null));
}
@Override
protected Long doGetOffset(Partition partition) {
return this.data.get(partition);
}
@Override
public void flush() throws IOException {
// not supported
}
@Override
public void close() throws IOException {
this.producer.close();
try {
((DefaultConnectionFactory) this.connectionFactory).destroy();
}
catch (Exception e) {
throw new IOException(e);
}
}
private void createCompactedTopicIfNotFound(ZkClient zkClient) {
Properties topicConfig = new Properties();
topicConfig.setProperty(CLEANUP_POLICY, CLEANUP_POLICY_COMPACT);
topicConfig.setProperty(DELETE_RETENTION, String.valueOf(this.retentionTime));
topicConfig.setProperty(SEGMENT_BYTES, String.valueOf(this.segmentSize));
try {
this.replicationFactor = 1;
AdminUtils$.MODULE$.createTopic(zkClient, this.topic, 1, this.replicationFactor, topicConfig);
}
catch (TopicExistsException e) {
log.debug("Topic already exists", e);
}
}
private void validateOffsetTopic(ZkClient zkClient) throws Exception {
//validate that the topic exists, but also prevent working with the topic until it's fully initialized
// set a retry template, since operations may fail
RetryTemplate retryValidateTopic = new RetryTemplate();
retryValidateTopic.setRetryPolicy(new SimpleRetryPolicy(10,
Collections.<Class<? extends Throwable>, Boolean>singletonMap(TopicNotFoundException.class, true)));
ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
backOffPolicy.setInitialInterval(100L);
backOffPolicy.setMaxInterval(1000L);
backOffPolicy.setMultiplier(2);
retryValidateTopic.setBackOffPolicy(backOffPolicy);
Collection<Partition> partitions =
retryValidateTopic.execute(new RetryCallback<Collection<Partition>, Exception>() {
@Override
public Collection<Partition> doWithRetry(RetryContext context) throws Exception {
return connectionFactory.getPartitions(topic);
}
});
if (partitions.size() > 1) {
throw new BeanInitializationException("Offset management topic cannot have more than one partition");
}
Properties properties = AdminUtils$.MODULE$.fetchTopicConfig(zkClient, this.topic);
if (!properties.containsKey(CLEANUP_POLICY)
|| !CLEANUP_POLICY_COMPACT.equals(properties.getProperty(CLEANUP_POLICY))) {
// we set the property to compact, but if using an already created topic,
// we must check if it is set up correctly
throw new BeanInitializationException("Property 'cleanup.policy' must be set to 'compact' on offset topic");
}
}
private void readOffsetData(Partition offsetManagementPartition, BrokerAddress leader) {
Result<Long> earliestOffsetResult = this.connectionFactory.connect(leader)
.fetchInitialOffset(OffsetRequest.EarliestTime(), offsetManagementPartition);
if (earliestOffsetResult.getErrors().size() > 0) {
throw new BeanInitializationException("Cannot initialize offset manager, unable to read earliest offset",
ErrorMapping$.MODULE$.exceptionFor(earliestOffsetResult.getError(offsetManagementPartition)));
}
Result<Long> latestOffsetResult = this.connectionFactory.connect(leader)
.fetchInitialOffset(OffsetRequest.LatestTime(), offsetManagementPartition);
if (latestOffsetResult.getErrors().size() > 0) {
throw new BeanInitializationException("Cannot initialize offset manager, unable to read latest offset");
}
long initialOffset = earliestOffsetResult.getResult(offsetManagementPartition);
long finalOffset = latestOffsetResult.getResult(offsetManagementPartition);
// read repeatedly until we drain the topic and add messages to the data map
long readingOffset = initialOffset;
while (readingOffset < finalOffset) {
FetchRequest fetchRequest = new FetchRequest(offsetManagementPartition, readingOffset, maxSize);
Result<KafkaMessageBatch> receive = this.kafkaTemplate.receive(Collections.singleton(fetchRequest));
if (receive.getErrors().size() > 0) {
throw new BeanInitializationException("Error while fetching initial offsets:",
ErrorMapping$.MODULE$.exceptionFor(receive.getError(offsetManagementPartition)));
}
KafkaMessageBatch result = receive.getResult(offsetManagementPartition);
for (KafkaMessage kafkaMessage : result.getMessages()) {
checkAndAddData(kafkaMessage);
readingOffset = kafkaMessage.getMetadata().getNextOffset();
}
if (log.isDebugEnabled()) {
log.debug(data.size() + " entries in the final map");
}
if (log.isTraceEnabled()) {
for (Map.Entry<Partition, Long> dataEntry : data.entrySet()) {
log.trace(String.format("Final value for %s : %s", dataEntry.getKey().toString(),
String.valueOf(dataEntry.getValue())));
}
}
}
}
private void checkAndAddData(KafkaMessage kafkaMessage) {
Key key = MessageUtils.decodeKey(kafkaMessage, KEY_CODEC);
Long value = MessageUtils.decodePayload(kafkaMessage, VALUE_CODEC);
if (log.isTraceEnabled()) {
log.trace("Loading key " + key + " with value " + value);
}
// we are only interested for messages that are intended for this consumer id
if (key != null && ObjectUtils.nullSafeEquals(this.consumerId, key.getConsumerId())) {
if (null != value) {
// write data in the cache, overwriting the older values
this.data.put(key.getPartition(), value);
}
else {
// a null value means that the data has been deleted, but not compacted yet
if (this.data.containsKey(key.getPartition())) {
this.data.remove(key.getPartition());
}
}
}
}
private void initializeProducer(BrokerAddress leader) throws Exception {
ProducerMetadata<Key,Long> producerMetadata = new ProducerMetadata<>(this.topic, Key.class, Long.class,
KEY_CODEC, VALUE_CODEC);
producerMetadata.setBatchBytes(batchBytes);
producerMetadata.setCompressionType(compressionType);
Properties additionalProps = new Properties();
additionalProps.setProperty(ProducerConfig.LINGER_MS_CONFIG, Integer.toString(maxQueueBufferingTime));
additionalProps.setProperty(ProducerConfig.ACKS_CONFIG, Integer.toString(requiredAcks));
ProducerFactoryBean<Key,Long> producerFactoryBean = new ProducerFactoryBean<>(producerMetadata,
leader.toString(), additionalProps);
this.producer = producerFactoryBean.getObject();
}
private static int intFromBytes(byte[] bytes, int start) {
return bytes[start] << 24 | (bytes[start + 1] & 0xFF) << 16
| (bytes[start + 2] & 0xFF) << 8 | (bytes[start + 3] & 0xFF);
}
private static byte[] intToBytes(Integer message) {
int value = message;
return new byte[] {
(byte) (value >>> 24),
(byte) (value >>> 16),
(byte) (value >>> 8),
(byte) value
};
}
/**
* Wraps the partition and consumer information and will be used as a key on the Kafka topic
*/
public static class Key {
String consumerId;
Partition partition;
public Key(String consumerID, Partition partition) {
Assert.notNull(consumerID, "Consumer Id cannot be null");
Assert.notNull(partition, "Partition cannot be null");
this.consumerId = consumerID;
this.partition = partition;
}
public String getConsumerId() {
return this.consumerId;
}
public Partition getPartition() {
return this.partition;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Key key = (Key) o;
return this.consumerId.equals(key.consumerId) && this.partition.equals(key.partition);
}
@Override
public int hashCode() {
int result = this.consumerId.hashCode();
result = 31 * result + this.partition.hashCode();
return result;
}
}
public static class KeySerializerDecoder implements Serializer<Key>, Decoder<Key> {
private static final Log log = LogFactory.getLog(KeySerializerDecoder.class);
@Override
public Key fromBytes(byte[] bytes) {
if (bytes == null || bytes.length <= 0) {
return null;
}
try {
// calculate the offsets in the key array
int consumerIdSize = intFromBytes(bytes, 0);
int topicIdSize = intFromBytes(bytes, consumerIdSize + 4);
// reconstruct the key
return new Key(new String(bytes, 4, consumerIdSize),
new Partition(new String(bytes, consumerIdSize + 8, topicIdSize),
intFromBytes(bytes, consumerIdSize + topicIdSize + 8)));
}
catch (Exception e) {
if (log.isDebugEnabled()) {
log.debug("Cannot decode key:" + LoggingUtils.asCommaSeparatedHexDump(bytes));
}
return null;
}
}
@Override
public void configure(Map<String, ?> configs, boolean isKey) {
// no-op
}
@Override
public byte[] serialize(String topic, Key data) {
if (data == null) {
return null;
}
try {
byte[] consumerIdBytes = data.consumerId.getBytes("UTF-8");
byte[] topicNameBytes = data.partition.getTopic().getBytes("UTF-8");
byte[] partitionIdBytes = intToBytes(data.partition.getId());
byte[] result = new byte[4 + consumerIdBytes.length + 4 + topicNameBytes.length + 4];
System.arraycopy(intToBytes(consumerIdBytes.length), 0, result, 0, 4);
System.arraycopy(consumerIdBytes, 0, result, 4, consumerIdBytes.length);
System.arraycopy(intToBytes(topicNameBytes.length), 0, result, consumerIdBytes.length + 4, 4);
System.arraycopy(topicNameBytes, 0, result, consumerIdBytes.length + 8, topicNameBytes.length);
System.arraycopy(partitionIdBytes, 0, result, consumerIdBytes.length + topicNameBytes.length + 8, 4);
return result;
}
catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void close() {
//no-op
}
}
}

38
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LoggingErrorHandler.java
View File

@@ -1,38 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.integration.kafka.core.KafkaMessage;
import org.springframework.util.ObjectUtils;
/**
* @author Marius Bogoevici
*/
public class LoggingErrorHandler implements ErrorHandler {
private static final Log log = LogFactory.getLog(LoggingErrorHandler.class);
@Override
public void handle(Exception thrownException, KafkaMessage message) {
log.error("Error while processing: " + ObjectUtils.nullSafeToString(message), thrownException);
}
}

76
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LongSerializerDecoder.java
View File

@@ -1,76 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import java.nio.ByteBuffer;
import java.util.Map;
import kafka.serializer.Decoder;
import kafka.serializer.Encoder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.kafka.common.serialization.Serializer;
import org.springframework.integration.kafka.util.LoggingUtils;
/**
* Kafka {@link Encoder} and {@link Decoder} for Long values.
*
* @author Marius Bogoevici
*/
public class LongSerializerDecoder implements Serializer<Long>, Decoder<Long> {
private Log log = LogFactory.getLog(LongSerializerDecoder.class);
@Override
public Long fromBytes(byte[] bytes) {
if (bytes == null || bytes.length <= 0) {
return null;
}
else {
try {
return ByteBuffer.wrap(bytes).getLong(0);
}
catch (Exception e) {
if (log.isDebugEnabled()) {
log.debug("Cannot decode value: " + LoggingUtils.asCommaSeparatedHexDump(bytes));
}
return null;
}
}
}
@Override
public void configure(Map<String, ?> configs, boolean isKey) {
// no-op
}
@Override
public byte[] serialize(String topic, Long data) {
if (data == null) {
return null;
}
else {
return ByteBuffer.allocate(8).putLong(data).array();
}
}
@Override
public void close() {
// no-op
}
}

35
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MessageListener.java
View File

@@ -1,35 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import org.springframework.integration.kafka.core.KafkaMessage;
/**
* Listener for handling incoming Kafka messages
*
* @author Marius Bogoevici
*/
public interface MessageListener {
/**
* Executes when a Kafka message is received
* @param message the Kafka message to be processed
*/
void onMessage(KafkaMessage message);
}

99
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MetadataStoreOffsetManager.java
View File

@@ -1,99 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import java.io.Closeable;
import java.io.Flushable;
import java.io.IOException;
import java.util.Map;
import org.springframework.integration.kafka.core.ConnectionFactory;
import org.springframework.integration.kafka.core.Partition;
import org.springframework.integration.metadata.MetadataStore;
import org.springframework.integration.metadata.SimpleMetadataStore;
/**
* An {@link OffsetManager} that persists offsets into a {@link MetadataStore}.
*
* @author Marius Bogoevici
*/
public class MetadataStoreOffsetManager extends AbstractOffsetManager {
private MetadataStore metadataStore = new SimpleMetadataStore();
public MetadataStoreOffsetManager(ConnectionFactory connectionFactory) {
super(connectionFactory);
}
public MetadataStoreOffsetManager(ConnectionFactory connectionFactory, Map<Partition, Long> initialOffsets) {
super(connectionFactory, initialOffsets);
}
/**
* The backing {@link MetadataStore} for storing offsets.
*
* @param metadataStore a fully configured {@link MetadataStore} instance
*/
public void setMetadataStore(MetadataStore metadataStore) {
this.metadataStore = metadataStore;
}
@Override
public void close() throws IOException {
// Flush before closing. This may be redundant, but ensures that the metadata store is closed properly
flush();
if (this.metadataStore instanceof Closeable) {
((Closeable) this.metadataStore).close();
}
}
@Override
public void flush() throws IOException {
if (this.metadataStore instanceof Flushable) {
((Flushable) this.metadataStore).flush();
}
}
@Override
protected void doUpdateOffset(Partition partition, long offset) {
this.metadataStore.put(generateKey(partition), Long.toString(offset));
}
@Override
protected void doRemoveOffset(Partition partition) {
this.metadataStore.remove(generateKey(partition));
}
protected Long doGetOffset(Partition partition) {
String storedOffsetValueAsString = this.metadataStore.get(generateKey(partition));
Long storedOffsetValue = null;
if (storedOffsetValueAsString != null) {
try {
storedOffsetValue = Long.parseLong(storedOffsetValueAsString);
}
catch (NumberFormatException e) {
log.warn("Invalid value: " + storedOffsetValueAsString);
}
}
return storedOffsetValue;
}
public String generateKey(Partition partition) {
return partition.getTopic() + ":" + partition.getId() + ":" + getConsumerId();
}
}

61
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/OffsetManager.java
View File

@@ -1,61 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import java.io.Closeable;
import java.io.Flushable;
import java.util.Collection;
import org.springframework.integration.kafka.core.Partition;
/**
* Stores and retrieves offsets for a Kafka consumer
*
* @author Marius Bogoevici
*/
public interface OffsetManager extends Closeable, Flushable {
/**
* Updates the offset for a given {@link Partition}
* @param partition the partition whose offset is to be updated
* @param offset the new offset value
*/
void updateOffset(Partition partition, long offset);
/**
* Retrieves the offset for a given {@link Partition}
* @param partition the partition to be
* @return the offset value
*/
long getOffset(Partition partition);
/**
* Removes the offset for a given {@link Partition}. Useful
* for components that need to clean up after themselves.
* @param partition for which to delete the JavaDoc
*
*/
void deleteOffset(Partition partition);
/**
* Resets offsets for the given {@link Partition}s. To be invoked when the values stored are invalid,
* so a client cannot resume from that position. Implementations must decide on the best strategy to follow.
* @param partition to reset
*/
void resetOffsets(Collection<Partition> partition);
}

264
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/QueueingMessageListenerInvoker.java
View File

@@ -1,264 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import java.util.List;
import java.util.concurrent.AbstractExecutorService;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.reactivestreams.Subscriber;
import org.reactivestreams.Subscription;
import org.springframework.core.task.support.ExecutorServiceAdapter;
import org.springframework.integration.kafka.core.KafkaMessage;
import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor;
import org.springframework.util.ObjectUtils;
import reactor.core.processor.RingBufferProcessor;
/**
* Invokes a delegate {@link MessageListener} for all the messages passed to it, storing
* them in an internal queue.
*
* @author Marius Bogoevici
* @author Stephane Maldini
*/
class QueueingMessageListenerInvoker {
private static Log log = LogFactory.getLog(QueueingMessageListenerInvoker.class);
private final MessageListener messageListener;
private final AcknowledgingMessageListener acknowledgingMessageListener;
private final OffsetManager offsetManager;
private final ErrorHandler errorHandler;
private final int capacity;
private final boolean autoCommitOnError;
private final ExecutorService executorService;
private volatile RingBufferProcessor<KafkaMessage> ringBufferProcessor;
private volatile CancelableSingleTaskExecutorService cancelableExecutorService;
private volatile boolean running = false;
private volatile CountDownLatch shutdownLatch;
public QueueingMessageListenerInvoker(int capacity, final OffsetManager offsetManager, Object delegate,
final ErrorHandler errorHandler, Executor executor, boolean autoCommitOnError) {
this.capacity = capacity;
this.autoCommitOnError = autoCommitOnError;
if (delegate instanceof MessageListener) {
this.messageListener = (MessageListener) delegate;
this.acknowledgingMessageListener = null;
}
else if (delegate instanceof AcknowledgingMessageListener) {
this.acknowledgingMessageListener = (AcknowledgingMessageListener) delegate;
this.messageListener = null;
}
else {
// it's neither, an exception will be thrown
throw new IllegalArgumentException("Either a "
+ MessageListener.class.getName() + " or a "
+ AcknowledgingMessageListener.class.getName() + " must be provided");
}
this.offsetManager = offsetManager;
this.errorHandler = errorHandler;
if (executor != null) {
this.executorService = new ExecutorServiceAdapter(new ConcurrentTaskExecutor(
executor));
}
else {
this.executorService = Executors.newSingleThreadExecutor();
}
}
/**
* Add a message to the queue, waiting if the queue has reached its maximum capacity.
*
* @param message the KafkaMessage to add
*/
public void enqueue(KafkaMessage message) {
if (this.running) {
ringBufferProcessor.onNext(message);
}
}
public synchronized void start() {
if (!this.running) {
this.running = true;
// wraps the executor, allowing for the interruption of the processing thread
// on shutdown without
// stopping the executor service (which may be injected)
cancelableExecutorService = new CancelableSingleTaskExecutorService(executorService);
this.ringBufferProcessor = RingBufferProcessor.share(cancelableExecutorService, capacity);
this.ringBufferProcessor.subscribe(new KafkaMessageDispatchingSubscriber());
}
}
public synchronized void stop(long stopTimeout) {
if (this.running) {
this.running = false;
if (ringBufferProcessor != null) {
// cancel the current task
shutdownLatch = new CountDownLatch(1);
cancelableExecutorService.cancelTask();
// allow the processor to complete, even if no more messages will be processed
ringBufferProcessor.onComplete();
ringBufferProcessor = null;
try {
shutdownLatch.await(stopTimeout, TimeUnit.MILLISECONDS);
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
finally {
shutdownLatch = null;
}
}
}
}
/**
* {@link ExecutorService} implementation that supports the execution of a single
* task, deferring to the wrapped instance. Allows for interrupting the
* {@link RingBufferProcessor}'s executing thread, in case it is blocking.
* @since 1.3
*/
private static class CancelableSingleTaskExecutorService extends
AbstractExecutorService {
private Future<?> submittedTask;
private final ExecutorService executor;
public CancelableSingleTaskExecutorService(ExecutorService executor) {
this.executor = executor;
}
@Override
public void execute(Runnable task) {
if (submittedTask == null) {
submittedTask = this.executor.submit(task);
}
else {
throw new IllegalArgumentException("Cannot submit more than one task");
}
}
@Override
public void shutdown() {
throw new IllegalStateException("Manual shutdown not supported");
}
@Override
public List<Runnable> shutdownNow() {
throw new IllegalStateException("Manual shutdown not supported");
}
@Override
public boolean isShutdown() {
return false;
}
@Override
public boolean isTerminated() {
return false;
}
@Override
public boolean awaitTermination(long timeout, TimeUnit unit)
throws InterruptedException {
throw new IllegalStateException("Manual termination not supported");
}
private void cancelTask() {
if (submittedTask != null) {
submittedTask.cancel(true);
submittedTask = null;
}
}
}
private class KafkaMessageDispatchingSubscriber implements Subscriber<KafkaMessage> {
@Override
public void onSubscribe(Subscription s) {
s.request(Long.MAX_VALUE);
}
@Override
public void onNext(KafkaMessage kafkaMessage) {
if (running) {
try {
if (messageListener != null) {
messageListener.onMessage(kafkaMessage);
offsetManager.updateOffset(kafkaMessage.getMetadata()
.getPartition(), kafkaMessage.getMetadata()
.getNextOffset());
}
else {
acknowledgingMessageListener.onMessage(kafkaMessage,
new DefaultAcknowledgment(offsetManager, kafkaMessage));
}
}
catch (Exception e) {
// we handle errors here so that we make sure that offsets are handled
// concurrently
if (errorHandler != null) {
errorHandler.handle(e, kafkaMessage);
if (autoCommitOnError) {
offsetManager.updateOffset(kafkaMessage.getMetadata()
.getPartition(), kafkaMessage.getMetadata()
.getNextOffset());
}
}
}
}
else {
if (log.isDebugEnabled()) {
log.debug("Message discarded on shutdown (no offsets have been committed): "
+ ObjectUtils.nullSafeToString(kafkaMessage));
}
}
}
@Override
public void onError(Throwable t) {
// ignore
}
@Override
public void onComplete() {
if (shutdownLatch != null) {
shutdownLatch.countDown();
}
}
}
}

310
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/WindowingOffsetManager.java
View File

@@ -1,310 +0,0 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.listener;
import java.io.IOException;
import java.util.Collection;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.integration.kafka.core.Partition;
import org.springframework.util.Assert;
import reactor.Environment;
import reactor.core.processor.RingBufferProcessor;
import reactor.fn.BiFunction;
import reactor.fn.Consumer;
import reactor.fn.Function;
import reactor.rx.Stream;
import reactor.rx.Streams;
import reactor.rx.stream.GroupedStream;
/**
* An {@link OffsetManager} that aggregates writes over a time or count window, using an underlying delegate to
* do the actual operations. Its purpose is to reduce the performance impact of writing operations
* wherever this is desirable.
* <p>
* A time window or a number of writes can be specified, or both.
* Defaults to 10 seconds window with {@link Integer#MAX_VALUE} buffer.
* @author Marius Bogoevici
* @author Artem Bilan
* @since 1.3.1
*/
public class WindowingOffsetManager implements OffsetManager, InitializingBean, DisposableBean {
static {
Environment.initializeIfEmpty();
}
private static final BiFunction<Long, Long, Long> maxFunction = new BiFunction<Long, Long, Long>() {
@Override
public Long apply(Long aLong, Long bLong) {
return Math.max(aLong, bLong);
}
};
private static final Function<PartitionAndOffset, Long> offsetFunction
= new Function<PartitionAndOffset, Long>() {
@Override
public Long apply(PartitionAndOffset partitionAndOffset) {
return partitionAndOffset.getOffset();
}
};
private static final ComputeMaximumOffsetByPartitionFunction findHighestOffsetInPartitionGroup
= new ComputeMaximumOffsetByPartitionFunction();
private static final Function<PartitionAndOffset, Partition> getPartitionFunction
= new Function<PartitionAndOffset, Partition>() {
@Override
public Partition apply(PartitionAndOffset partitionAndOffset) {
return partitionAndOffset.getPartition();
}
};
private static final FindHighestOffsetsByPartitionFunction findHighestOffsetsByPartition
= new FindHighestOffsetsByPartitionFunction();
private final Consumer<PartitionAndOffset> delegateUpdateOffset = new Consumer<PartitionAndOffset>() {
@Override
public void accept(PartitionAndOffset partitionAndOffset) {
delegate.updateOffset(partitionAndOffset.getPartition(), partitionAndOffset.getOffset());
}
};
private final Consumer<Void> offsetComplete = new Consumer<Void>() {
@Override
public void accept(Void aVoid) {
createOffsetsStream();
}
};
private final ReadWriteLock offsetsLock = new ReentrantReadWriteLock();
private final OffsetManager delegate;
private long timespan = 10 * 1000;
private int count = Integer.MAX_VALUE;
private int shutdownTimeout = 2000;
private volatile RingBufferProcessor<PartitionAndOffset> offsets;
private volatile boolean closed;
public WindowingOffsetManager(OffsetManager offsetManager) {
this.delegate = offsetManager;
}
/**
* The timespan for aggregating write operations, before invoking the underlying {@link OffsetManager}.
* @param timespan duration in milliseconds
*/
public void setTimespan(long timespan) {
Assert.isTrue(timespan >= 0, "Timespan must be a positive value");
this.timespan = timespan;
}
/**
* How many writes should be aggregated, before invoking the underlying {@link OffsetManager}. Setting this value
* to 1 effectively disables windowing.
* @param count number of writes
*/
public void setCount(int count) {
Assert.isTrue(count >= 0, "Count must be a positive value");
this.count = count;
}
/**
* The timeout that {@link #close()} and {@link #destroy()}
* operations will wait for receiving a confirmation that
* the underlying writes have been processed.
* @param shutdownTimeout duration in milliseconds
*/
public void setShutdownTimeout(int shutdownTimeout) {
this.shutdownTimeout = shutdownTimeout;
}
@Override
public void afterPropertiesSet() throws Exception {
if (this.count != 1) {
createOffsetsStream();
}
}
private void createOffsetsStream() {
if (!this.closed) {
this.offsetsLock.writeLock().lock();
try {
this.offsets = RingBufferProcessor.share("spring-integration-kafka-offset", 1024);
}
finally {
this.offsetsLock.writeLock().unlock();
}
Streams.wrap(this.offsets)
.window(this.count, timespan, TimeUnit.MILLISECONDS)
.flatMap(findHighestOffsetsByPartition)
.consume(this.delegateUpdateOffset, null, this.offsetComplete);
}
}
@Override
public void destroy() throws Exception {
flush();
close();
if (this.delegate instanceof DisposableBean) {
((DisposableBean) this.delegate).destroy();
}
}
@Override
public void updateOffset(Partition partition, long offset) {
if (this.offsets != null) {
this.offsetsLock.readLock().lock();
try {
this.offsets.onNext(new PartitionAndOffset(partition, offset));
}
finally {
this.offsetsLock.readLock().unlock();
}
}
else {
this.delegate.updateOffset(partition, offset);
}
}
@Override
public long getOffset(Partition partition) {
doFlush();
return this.delegate.getOffset(partition);
}
@Override
public void deleteOffset(Partition partition) {
doFlush();
this.delegate.deleteOffset(partition);
}
@Override
public void resetOffsets(Collection<Partition> partition) {
doFlush();
this.delegate.resetOffsets(partition);
}
@Override
public void close() throws IOException {
this.closed = true;
this.delegate.close();
}
@Override
public void flush() throws IOException {
if (this.offsets != null) {
this.offsets.awaitAndShutdown(this.shutdownTimeout, TimeUnit.MILLISECONDS);
}
this.delegate.flush();
}
private void doFlush() {
try {
flush();
}
catch (IOException e) {
throw new IllegalStateException(e);
}
}
private static class PartitionAndOffset {
private final Partition partition;
private final Long offset;
public PartitionAndOffset(Partition partition, Long offset) {
this.partition = partition;
this.offset = offset;
}
public Partition getPartition() {
return partition;
}
public Long getOffset() {
return offset;
}
@Override
public String toString() {
return "PartitionAndOffset{" +
"partition=" + partition +
", offset=" + offset +
'}';
}
}
private static class ComputeMaximumOffsetByPartitionFunction
implements Function<GroupedStream<Partition, PartitionAndOffset>, Stream<PartitionAndOffset>> {
@Override
public Stream<PartitionAndOffset> apply(final GroupedStream<Partition, PartitionAndOffset> group) {
return group
.map(offsetFunction)
.reduce(maxFunction)
.map(new Function<Long, PartitionAndOffset>() {
@Override
public PartitionAndOffset apply(Long offset) {
return new PartitionAndOffset(group.key(), offset);
}
});
}
}
private static class FindHighestOffsetsByPartitionFunction
implements Function<Stream<PartitionAndOffset>, Stream<PartitionAndOffset>> {
@Override
public Stream<PartitionAndOffset> apply(Stream<PartitionAndOffset> windowBuffer) {
return windowBuffer
.groupBy(getPartitionFunction)
.flatMap(findHighestOffsetInPartitionGroup);
}
}
}

4
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/package-info.java
View File

@@ -1,4 +0,0 @@
/**
* Provides message listener container support
*/
package org.springframework.integration.kafka.listener;

27
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/outbound/KafkaProducerMessageHandler.java
View File

@@ -20,9 +20,10 @@ import org.springframework.expression.EvaluationContext;
import org.springframework.expression.Expression;
import org.springframework.integration.expression.ExpressionUtils;
import org.springframework.integration.handler.AbstractMessageHandler;
import org.springframework.integration.kafka.support.KafkaHeaders;
import org.springframework.integration.kafka.support.KafkaProducerContext;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.KafkaHeaders;
import org.springframework.messaging.Message;
import org.springframework.util.Assert;
/**
* @author Soby Chacko
@@ -31,9 +32,9 @@ import org.springframework.messaging.Message;
* @author Marius Bogoevici
* @since 0.5
*/
public class KafkaProducerMessageHandler extends AbstractMessageHandler {
public class KafkaProducerMessageHandler<K, V> extends AbstractMessageHandler {
private final KafkaProducerContext kafkaProducerContext;
private final KafkaTemplate<K, V> kafkaTemplate;
private EvaluationContext evaluationContext;
@@ -45,8 +46,9 @@ public class KafkaProducerMessageHandler extends AbstractMessageHandler {
private volatile Expression partitionIdExpression;
public KafkaProducerMessageHandler(final KafkaProducerContext kafkaProducerContext) {
this.kafkaProducerContext = kafkaProducerContext;
public KafkaProducerMessageHandler(final KafkaTemplate<K, V> kafkaTemplate) {
Assert.notNull(kafkaTemplate, "kafkaTemplate cannot be null");
this.kafkaTemplate = kafkaTemplate;
}
/**
@@ -83,8 +85,8 @@ public class KafkaProducerMessageHandler extends AbstractMessageHandler {
this.partitionIdExpression = partitionIdExpression;
}
public KafkaProducerContext getKafkaProducerContext() {
return this.kafkaProducerContext;
public KafkaTemplate<?, ?> getKafkaTemplate() {
return this.kafkaTemplate;
}
@Override
@@ -93,6 +95,7 @@ public class KafkaProducerMessageHandler extends AbstractMessageHandler {
this.evaluationContext = ExpressionUtils.createStandardEvaluationContext(getBeanFactory());
}
@SuppressWarnings("unchecked")
@Override
protected void handleMessageInternal(final Message<?> message) throws Exception {
String topic = this.topicExpression != null ?
@@ -108,7 +111,13 @@ public class KafkaProducerMessageHandler extends AbstractMessageHandler {
? this.messageKeyExpression.getValue(this.evaluationContext, message)
: message.getHeaders().get(KafkaHeaders.MESSAGE_KEY);
this.kafkaProducerContext.send(topic, partitionId, messageKey, message.getPayload());
// TODO: Add KafkaTemplate method with topic, partition, data only (no key)
if (partitionId == null) {
this.kafkaTemplate.convertAndSend(topic, (K) messageKey, ((V) message.getPayload()));
}
else {
this.kafkaTemplate.convertAndSend(topic, partitionId, (K) messageKey, ((V) message.getPayload()));
}
}
@Override

41
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroDatumSupport.java
View File

@@ -1,41 +0,0 @@
package org.springframework.integration.kafka.serializer.avro;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.io.IOException;
/**
* @author Soby Chacko
* @since 0.5
*/
public abstract class AvroDatumSupport<T> {
private static final Log LOG = LogFactory.getLog(AvroDatumSupport.class);
private final AvroSerializer<T> avroSerializer;
protected AvroDatumSupport() {
this.avroSerializer = new AvroSerializer<T>();
}
public byte[] toBytes(final T source, final DatumWriter<T> writer) {
try {
return avroSerializer.serialize(source, writer);
} catch (IOException e) {
LOG.error("Failed to encode source: " + e);
}
return null;
}
public T fromBytes(final byte[] bytes, final DatumReader<T> reader) {
try {
return avroSerializer.deserialize(bytes, reader);
} catch (IOException e) {
LOG.error("Failed to decode byte array: " + e);
}
return null;
}
}

39
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroReflectDatumBackedKafkaDecoder.java
View File

@@ -1,39 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.serializer.avro;
import kafka.serializer.Decoder;
import org.apache.avro.io.DatumReader;
import org.apache.avro.reflect.ReflectDatumReader;
/**
* @author Soby Chacko
* @since 0.5
*/
public class AvroReflectDatumBackedKafkaDecoder<T> extends AvroDatumSupport<T> implements Decoder<T> {
private final DatumReader<T> reader;
public AvroReflectDatumBackedKafkaDecoder(final Class<T> clazz) {
this.reader = new ReflectDatumReader<T>(clazz);
}
@Override
public T fromBytes(final byte[] bytes) {
return fromBytes(bytes, reader);
}
}

39
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroReflectDatumBackedKafkaEncoder.java
View File

@@ -1,39 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.serializer.avro;
import kafka.serializer.Encoder;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.reflect.ReflectDatumWriter;
/**
* @author Soby Chacko
* @since 0.5
*/
public class AvroReflectDatumBackedKafkaEncoder<T> extends AvroDatumSupport<T> implements Encoder<T> {
private final DatumWriter<T> writer;
public AvroReflectDatumBackedKafkaEncoder(final Class<T> clazz) {
this.writer = new ReflectDatumWriter<T>(clazz);
}
@Override
public byte[] toBytes(final T source) {
return toBytes(source, writer);
}
}

48
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSerializer.java
View File

@@ -1,48 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.serializer.avro;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.Encoder;
import org.apache.avro.io.EncoderFactory;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
/**
* @author Soby Chacko
* @since 0.5
*/
public class AvroSerializer<T> {
public T deserialize(final byte[] bytes, final DatumReader<T> reader) throws IOException {
final Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null);
return reader.read(null, decoder);
}
public byte[] serialize(final T input, final DatumWriter<T> writer) throws IOException {
final ByteArrayOutputStream stream = new ByteArrayOutputStream();
final Encoder encoder = EncoderFactory.get().binaryEncoder(stream, null);
writer.write(input, encoder);
encoder.flush();
return stream.toByteArray();
}
}

24
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumBackedKafkaDecoder.java
View File

@@ -1,24 +0,0 @@
package org.springframework.integration.kafka.serializer.avro;
import kafka.serializer.Decoder;
import org.apache.avro.io.DatumReader;
import org.apache.avro.specific.SpecificDatumReader;
/**
* @author Soby Chacko
* @since 0.5
*/
public class AvroSpecificDatumBackedKafkaDecoder<T> extends AvroDatumSupport<T> implements Decoder<T> {
private final DatumReader<T> reader;
public AvroSpecificDatumBackedKafkaDecoder(final Class<T> specificRecordBase) {
this.reader = new SpecificDatumReader<T>(specificRecordBase);
}
@Override
public T fromBytes(final byte[] bytes) {
return fromBytes(bytes, reader);
}
}

24
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumBackedKafkaEncoder.java
View File

@@ -1,24 +0,0 @@
package org.springframework.integration.kafka.serializer.avro;
import kafka.serializer.Encoder;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumWriter;
/**
* @author Soby Chacko
* @since 0.5
*/
public class AvroSpecificDatumBackedKafkaEncoder<T> extends AvroDatumSupport<T> implements Encoder<T> {
private final DatumWriter<T> writer;
public AvroSpecificDatumBackedKafkaEncoder(final Class<T> specificRecordClazz) {
this.writer = new SpecificDatumWriter<T>(specificRecordClazz);
}
@Override
public byte[] toBytes(final T source) {
return toBytes(source, writer);
}
}

50
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringDecoder.java
View File

@@ -1,50 +0,0 @@
/*
* Copyright 2002-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.serializer.common;
import java.util.Properties;
import kafka.serializer.Decoder;
import kafka.utils.VerifiableProperties;
/**
* String Decoder for Kafka message key/value decoding.
* The Default decoder returns the same byte array it takes in.
*
* @author Soby Chacko
* @author Ilayaperumal Gopinathan
*/
public class StringDecoder implements Decoder<String> {
private final kafka.serializer.StringDecoder stringDecoder;
public StringDecoder() {
this("UTF8");
}
public StringDecoder(final String encoding) {
final Properties props = new Properties();
props.put("serializer.encoding", encoding);
this.stringDecoder = new kafka.serializer.StringDecoder(new VerifiableProperties(props));
}
@Override
public String fromBytes(byte[] bytes) {
return this.stringDecoder.fromBytes(bytes);
}
}

47
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringEncoder.java
View File

@@ -1,47 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.serializer.common;
import java.util.Properties;
import kafka.serializer.Encoder;
import kafka.utils.VerifiableProperties;
/**
* @author Soby Chacko
* @author Marius Bogoevici
* @since 0.5
*/
public class StringEncoder implements Encoder<String> {
private kafka.serializer.StringEncoder stringEncoder;
public StringEncoder() {
this("UTF-8");
}
public StringEncoder(String encoding) {
final Properties props = new Properties();
props.put("serializer.encoding", encoding);
final VerifiableProperties verifiableProperties = new VerifiableProperties(props);
stringEncoder = new kafka.serializer.StringEncoder(verifiableProperties);
}
@Override
public byte[] toBytes(final String value) {
return stringEncoder.toBytes(value);
}
}

85
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfigFactoryBean.java
View File

@@ -1,85 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.FactoryBean;
import kafka.consumer.ConsumerConfig;
/**
* @author Soby Chacko
* @since 0.5
*/
@Deprecated
@SuppressWarnings("deprecation")
public class ConsumerConfigFactoryBean<K,V> implements FactoryBean<ConsumerConfig> {
private static final Log LOGGER = LogFactory.getLog(ConsumerConfigFactoryBean.class);
private final ConsumerMetadata<K,V> consumerMetadata;
private final ZookeeperConnect zookeeperConnect;
private Properties consumerProperties = new Properties();
public ConsumerConfigFactoryBean(final ConsumerMetadata<K,V> consumerMetadata,
final ZookeeperConnect zookeeperConnect, final Properties consumerProperties) {
this.consumerMetadata = consumerMetadata;
this.zookeeperConnect = zookeeperConnect;
if (consumerProperties != null) {
this.consumerProperties = consumerProperties;
}
}
public ConsumerConfigFactoryBean(final ConsumerMetadata<K, V> consumerMetadata,
final ZookeeperConnect zookeeperConnect) {
this(consumerMetadata, zookeeperConnect, null);
}
@Override
public ConsumerConfig getObject() throws Exception {
final Properties properties = new Properties();
properties.putAll(consumerProperties);
properties.put("zookeeper.connect", zookeeperConnect.getZkConnect());
properties.put("zookeeper.session.timeout.ms", zookeeperConnect.getZkSessionTimeout());
properties.put("zookeeper.sync.time.ms", zookeeperConnect.getZkSyncTime());
// Overriding the default value of -1, which will make the consumer to
// wait indefinitely
if (!properties.containsKey("consumer.timeout.ms")) {
properties.put("consumer.timeout.ms", consumerMetadata.getConsumerTimeout());
}
properties.put("group.id", consumerMetadata.getGroupId());
LOGGER.info("Using consumer properties => " + properties);
return new ConsumerConfig(properties);
}
@Override
public Class<?> getObjectType() {
return ConsumerConfig.class;
}
@Override
public boolean isSingleton() {
return true;
}
}

280
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfiguration.java
View File

@@ -1,280 +0,0 @@
/*
* Copyright 2002-2015 the original author or authors. Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language governing permissions and limitations under the
* License.
*/
package org.springframework.integration.kafka.support;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.messaging.MessagingException;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.util.Assert;
import kafka.consumer.ConsumerTimeoutException;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.message.MessageAndMetadata;
/**
* @author Soby Chacko
* @author Rajasekar Elango
* @author Artem Bilan
* @since 0.5
*/
@Deprecated
@SuppressWarnings("deprecation")
public class ConsumerConfiguration<K, V> {
private static final Log LOGGER = LogFactory.getLog(ConsumerConfiguration.class);
private final ConsumerMetadata<K, V> consumerMetadata;
private final ConsumerConnectionProvider consumerConnectionProvider;
private final MessageLeftOverTracker<K, V> messageLeftOverTracker;
private ConsumerConnector consumerConnector;
private volatile int count = 0;
private int maxMessages = 1;
private Collection<List<KafkaStream<K, V>>> consumerMessageStreams;
private ExecutorService executorService = Executors.newCachedThreadPool();
private boolean executorExplicitlySet;
private volatile boolean stopped;
public ConsumerConfiguration(final ConsumerMetadata<K, V> consumerMetadata,
final ConsumerConnectionProvider consumerConnectionProvider,
final MessageLeftOverTracker<K, V> messageLeftOverTracker) {
this.consumerMetadata = consumerMetadata;
this.consumerConnectionProvider = consumerConnectionProvider;
this.messageLeftOverTracker = messageLeftOverTracker;
}
public void setExecutor(Executor executor) {
boolean isExecutorService = executor instanceof ExecutorService;
boolean isThreadPoolTaskExecutor = executor instanceof ThreadPoolTaskExecutor;
Assert.isTrue(isExecutorService || isThreadPoolTaskExecutor);
if (isExecutorService) {
this.executorService = (ExecutorService) executor;
}
else {
this.executorService = ((ThreadPoolTaskExecutor) executor).getThreadPoolExecutor();
}
this.executorExplicitlySet = true;
}
public ConsumerMetadata<K, V> getConsumerMetadata() {
return consumerMetadata;
}
public Map<String, Map<Integer, List<Object>>> receive() {
count = messageLeftOverTracker.getCurrentCount();
final Object lock = new Object();
final List<Callable<List<MessageAndMetadata<K, V>>>> tasks = new LinkedList<Callable<List<MessageAndMetadata<K, V>>>>();
for (final List<KafkaStream<K, V>> streams : createConsumerMessageStreams()) {
for (final KafkaStream<K, V> stream : streams) {
tasks.add(new Callable<List<MessageAndMetadata<K, V>>>() {
@Override
public List<MessageAndMetadata<K, V>> call() throws Exception {
final List<MessageAndMetadata<K, V>> rawMessages = new ArrayList<MessageAndMetadata<K, V>>();
try {
while (count < maxMessages) {
final MessageAndMetadata<K, V> messageAndMetadata = stream.iterator().next();
synchronized (lock) {
if (count < maxMessages) {
rawMessages.add(messageAndMetadata);
count++;
}
else {
messageLeftOverTracker.addMessageAndMetadata(messageAndMetadata);
}
}
}
}
catch (ConsumerTimeoutException cte) {
LOGGER.debug("Consumer timed out");
}
return rawMessages;
}
});
}
}
return executeTasks(tasks);
}
private Map<String, Map<Integer, List<Object>>> executeTasks(
final List<Callable<List<MessageAndMetadata<K, V>>>> tasks) {
final Map<String, Map<Integer, List<Object>>> messages = new ConcurrentHashMap<String, Map<Integer, List<Object>>>();
messages.putAll(getLeftOverMessageMap());
try {
for (final Future<List<MessageAndMetadata<K, V>>> result : this.executorService.invokeAll(tasks)) {
if (!result.get().isEmpty()) {
final String topic = result.get().get(0).topic();
if (!messages.containsKey(topic)) {
messages.put(topic, getPayload(result.get()));
}
else {
final Map<Integer, List<Object>> existingPayloadMap = messages.get(topic);
getPayload(result.get(), existingPayloadMap);
}
}
}
}
catch (Exception e) {
if (!this.stopped) {
throw new MessagingException("Consuming from Kafka failed", e);
}
else {
LOGGER.warn("Consuming from Kafka failed", e);
}
}
if (messages.isEmpty()) {
return null;
}
return messages;
}
private Map<String, Map<Integer, List<Object>>> getLeftOverMessageMap() {
final Map<String, Map<Integer, List<Object>>> messages = new ConcurrentHashMap<String, Map<Integer, List<Object>>>();
for (final MessageAndMetadata<K, V> mamd : messageLeftOverTracker.getMessageLeftOverFromPreviousPoll()) {
final String topic = mamd.topic();
if (!messages.containsKey(topic)) {
final List<MessageAndMetadata<K, V>> l = new ArrayList<MessageAndMetadata<K, V>>();
l.add(mamd);
messages.put(topic, getPayload(l));
}
else {
final Map<Integer, List<Object>> existingPayloadMap = messages.get(topic);
final List<MessageAndMetadata<K, V>> l = new ArrayList<MessageAndMetadata<K, V>>();
l.add(mamd);
getPayload(l, existingPayloadMap);
}
}
messageLeftOverTracker.clearMessagesLeftOver();
return messages;
}
private Map<Integer, List<Object>> getPayload(final List<MessageAndMetadata<K, V>> messageAndMetadatas) {
final Map<Integer, List<Object>> payloadMap = new ConcurrentHashMap<Integer, List<Object>>();
for (final MessageAndMetadata<K, V> messageAndMetadata : messageAndMetadatas) {
if (!payloadMap.containsKey(messageAndMetadata.partition())) {
final List<Object> payload = new ArrayList<Object>();
payload.add(messageAndMetadata.message());
payloadMap.put(messageAndMetadata.partition(), payload);
}
else {
final List<Object> payload = payloadMap.get(messageAndMetadata.partition());
payload.add(messageAndMetadata.message());
}
}
return payloadMap;
}
private void getPayload(final List<MessageAndMetadata<K, V>> messageAndMetadatas,
final Map<Integer, List<Object>> existingPayloadMap) {
for (final MessageAndMetadata<K, V> messageAndMetadata : messageAndMetadatas) {
if (!existingPayloadMap.containsKey(messageAndMetadata.partition())) {
final List<Object> payload = new ArrayList<Object>();
payload.add(messageAndMetadata.message());
existingPayloadMap.put(messageAndMetadata.partition(), payload);
}
else {
final List<Object> payload = existingPayloadMap.get(messageAndMetadata.partition());
payload.add(messageAndMetadata.message());
}
}
}
private Collection<List<KafkaStream<K, V>>> createConsumerMessageStreams() {
if (consumerMessageStreams == null) {
if (!(consumerMetadata.getTopicStreamMap() == null || consumerMetadata.getTopicStreamMap().isEmpty())) {
consumerMessageStreams = createMessageStreamsForTopic().values();
}
else {
consumerMessageStreams = new ArrayList<List<KafkaStream<K, V>>>();
consumerMessageStreams.add(createMessageStreamsForTopicFilter());
}
}
return consumerMessageStreams;
}
public Map<String, List<KafkaStream<K, V>>> createMessageStreamsForTopic() {
return getConsumerConnector().createMessageStreams(consumerMetadata.getTopicStreamMap(),
consumerMetadata.getKeyDecoder(), consumerMetadata.getValueDecoder());
}
public List<KafkaStream<K, V>> createMessageStreamsForTopicFilter() {
List<KafkaStream<K, V>> messageStream = new ArrayList<KafkaStream<K, V>>();
TopicFilterConfiguration topicFilterConfiguration = consumerMetadata.getTopicFilterConfiguration();
if (topicFilterConfiguration != null) {
messageStream = getConsumerConnector().createMessageStreamsByFilter(
topicFilterConfiguration.getTopicFilter(), topicFilterConfiguration.getNumberOfStreams(),
consumerMetadata.getKeyDecoder(), consumerMetadata.getValueDecoder());
}
else {
LOGGER.warn("No Topic Filter Configuration defined");
}
return messageStream;
}
public int getMaxMessages() {
return maxMessages;
}
public void setMaxMessages(final int maxMessages) {
this.maxMessages = maxMessages;
}
public ConsumerConnector getConsumerConnector() {
if (consumerConnector == null) {
consumerConnector = consumerConnectionProvider.getConsumerConnector();
}
return consumerConnector;
}
public void shutdown() {
this.stopped = true;
if (!this.executorExplicitlySet) {
this.executorService.shutdownNow();
}
getConsumerConnector().shutdown();
}
}

37
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConnectionProvider.java
View File

@@ -1,37 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import kafka.consumer.ConsumerConfig;
import kafka.javaapi.consumer.ConsumerConnector;
/**
* @author Soby Chacko
* @since 0.5
*/
@Deprecated
public class ConsumerConnectionProvider {
private final ConsumerConfig consumerConfig;
public ConsumerConnectionProvider(final ConsumerConfig consumerConfig) {
this.consumerConfig = consumerConfig;
}
public ConsumerConnector getConsumerConnector() {
return kafka.consumer.Consumer.createJavaConsumerConnector(consumerConfig);
}
}

203
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerMetadata.java
View File

@@ -1,203 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import java.util.Map;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.integration.kafka.core.KafkaConsumerDefaults;
import kafka.serializer.Decoder;
import kafka.serializer.DefaultDecoder;
/**
* @author Soby Chacko
* @author Rajasekar Elango
* @since 0.5
*/
@Deprecated
public class ConsumerMetadata<K,V> implements InitializingBean {
//High level consumer defaults
private String groupId = KafkaConsumerDefaults.GROUP_ID;
private String socketTimeout = KafkaConsumerDefaults.SOCKET_TIMEOUT;
private String socketBufferSize = KafkaConsumerDefaults.SOCKET_BUFFER_SIZE;
private String fetchSize = KafkaConsumerDefaults.FETCH_SIZE;
private String backoffIncrement = KafkaConsumerDefaults.BACKOFF_INCREMENT;
private String queuedChunksMax = KafkaConsumerDefaults.QUEUED_CHUNKS_MAX;
private String autoCommitEnable = KafkaConsumerDefaults.AUTO_COMMIT_ENABLE;
private String autoCommitInterval = KafkaConsumerDefaults.AUTO_COMMIT_INTERVAL;
private String autoOffsetReset = KafkaConsumerDefaults.AUTO_OFFSET_RESET;
private String rebalanceRetriesMax = KafkaConsumerDefaults.REBALANCE_RETRIES_MAX;
private String consumerTimeout = KafkaConsumerDefaults.CONSUMER_TIMEOUT;
private String topic;
private int streams;
private Decoder<V> valueDecoder;
private Decoder<K> keyDecoder;
private Map<String, Integer> topicStreamMap;
private TopicFilterConfiguration topicFilterConfiguration;
public String getGroupId() {
return groupId;
}
public void setGroupId(final String groupId) {
this.groupId = groupId;
}
public String getSocketTimeout() {
return socketTimeout;
}
public void setSocketTimeout(final String socketTimeout) {
this.socketTimeout = socketTimeout;
}
public String getSocketBufferSize() {
return socketBufferSize;
}
public void setSocketBufferSize(final String socketBufferSize) {
this.socketBufferSize = socketBufferSize;
}
public String getFetchSize() {
return fetchSize;
}
public void setFetchSize(final String fetchSize) {
this.fetchSize = fetchSize;
}
public String getBackoffIncrement() {
return backoffIncrement;
}
public void setBackoffIncrement(final String backoffIncrement) {
this.backoffIncrement = backoffIncrement;
}
public String getQueuedChunksMax() {
return queuedChunksMax;
}
public void setQueuedChunksMax(final String queuedChunksMax) {
this.queuedChunksMax = queuedChunksMax;
}
public String getAutoCommitEnable() {
return autoCommitEnable;
}
public void setAutoCommitEnable(final String autoCommitEnable) {
this.autoCommitEnable = autoCommitEnable;
}
public String getAutoCommitInterval() {
return autoCommitInterval;
}
public void setAutoCommitInterval(final String autoCommitInterval) {
this.autoCommitInterval = autoCommitInterval;
}
public String getAutoOffsetReset() {
return autoOffsetReset;
}
public void setAutoOffsetReset(final String autoOffsetReset) {
this.autoOffsetReset = autoOffsetReset;
}
public String getRebalanceRetriesMax() {
return rebalanceRetriesMax;
}
public void setRebalanceRetriesMax(final String rebalanceRetriesMax) {
this.rebalanceRetriesMax = rebalanceRetriesMax;
}
public String getConsumerTimeout() {
return consumerTimeout;
}
public void setConsumerTimeout(final String consumerTimeout) {
this.consumerTimeout = consumerTimeout;
}
public String getTopic() {
return topic;
}
public void setTopic(final String topic) {
this.topic = topic;
}
public int getStreams() {
return streams;
}
public void setStreams(final int streams) {
this.streams = streams;
}
public Decoder<V> getValueDecoder() {
return valueDecoder;
}
public void setValueDecoder(final Decoder<V> valueDecoder) {
this.valueDecoder = valueDecoder;
}
public Decoder<K> getKeyDecoder() {
return keyDecoder;
}
public void setKeyDecoder(final Decoder<K> keyDecoder) {
this.keyDecoder = keyDecoder;
}
public Map<String, Integer> getTopicStreamMap() {
return topicStreamMap;
}
public void setTopicStreamMap(final Map<String, Integer> topicStreamMap) {
this.topicStreamMap = topicStreamMap;
}
@Override
@SuppressWarnings("unchecked")
public void afterPropertiesSet() throws Exception {
if (valueDecoder == null) {
setValueDecoder((Decoder<V>) new DefaultDecoder(null));
}
if (keyDecoder == null) {
setKeyDecoder((Decoder<K>) getValueDecoder());
}
}
public TopicFilterConfiguration getTopicFilterConfiguration() {
return topicFilterConfiguration;
}
public void setTopicFilterConfiguration(
TopicFilterConfiguration topicFilterConfiguration) {
this.topicFilterConfiguration = topicFilterConfiguration;
}
}

39
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultPartitioner.java
View File

@@ -1,39 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import kafka.producer.Partitioner;
import kafka.utils.Utils;
/**
* @author Soby Chacko
* @since 0.5
*
* This class is for internal use only and therefore is at default access level
*/
@Deprecated
class DefaultPartitioner implements Partitioner {
/**
* Uses the key to calculate a partition bucket id for routing
* the data to the appropriate broker partition
* @return an integer between 0 and numPartitions-1
*/
@Override
public int partition(final Object key, final int numPartitions) {
return Utils.abs(key.hashCode()) % numPartitions;
}
}

36
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultProducerListener.java
View File

@@ -1,36 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import org.apache.kafka.clients.producer.RecordMetadata;
/**
* No-op implementation of @link ProducerListener}, to be used as base class for other implementations.
*
* @author Marius Bogoevici
* @since 1.3
*/
public class DefaultProducerListener implements ProducerListener {
@Override
public void onSuccess(String topic, Integer partition, Object key, Object value, RecordMetadata recordMetadata) {
}
@Override
public void onError(String topic, Integer partition, Object key, Object value, Exception exception) {
}
}

92
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaConsumerContext.java
View File

@@ -1,92 +0,0 @@
/*
* Copyright 2002-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.integration.kafka.core.KafkaConsumerDefaults;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.messaging.Message;
import org.springframework.util.CollectionUtils;
/**
* @author Soby Chacko
* @author Ilayaperumal Gopinathan
* @since 0.5
*/
@Deprecated
@SuppressWarnings("deprecation")
public class KafkaConsumerContext<K, V> implements DisposableBean {
private Map<String, ConsumerConfiguration<K, V>> consumerConfigurations;
private String consumerTimeout = KafkaConsumerDefaults.CONSUMER_TIMEOUT;
private ZookeeperConnect zookeeperConnect;
public String getConsumerTimeout() {
return this.consumerTimeout;
}
public void setConsumerTimeout(final String consumerTimeout) {
this.consumerTimeout = consumerTimeout;
}
public ZookeeperConnect getZookeeperConnect() {
return this.zookeeperConnect;
}
public void setZookeeperConnect(final ZookeeperConnect zookeeperConnect) {
this.zookeeperConnect = zookeeperConnect;
}
public void setConsumerConfigurations(Map<String, ConsumerConfiguration<K, V>> consumerConfigurations) {
this.consumerConfigurations = consumerConfigurations;
}
public Map<String, ConsumerConfiguration<K, V>> getConsumerConfigurations() {
return this.consumerConfigurations;
}
public ConsumerConfiguration<K, V> getConsumerConfiguration(String groupId) {
return this.consumerConfigurations.get(groupId);
}
public Message<Map<String, Map<Integer, List<Object>>>> receive() {
final Map<String, Map<Integer, List<Object>>> consumedData = new HashMap<String, Map<Integer, List<Object>>>();
for (final ConsumerConfiguration<K, V> consumerConfiguration : getConsumerConfigurations().values()) {
final Map<String, Map<Integer, List<Object>>> messages = consumerConfiguration.receive();
if (!CollectionUtils.isEmpty(messages)) {
consumedData.putAll(messages);
}
}
return consumedData.isEmpty() ? null : MessageBuilder.withPayload(consumedData).build();
}
@Override
public void destroy() throws Exception {
for (ConsumerConfiguration<K, V> config : this.consumerConfigurations.values()) {
config.shutdown();
}
}
}

40
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaHeaders.java
View File

@@ -1,40 +0,0 @@
/*
* Copyright 2014-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
/**
* @author Artem Bilan
* @author Marius Bogoevici
* @since 1.0
*/
public abstract class KafkaHeaders {
private static final String PREFIX = "kafka_";
public static final String TOPIC = PREFIX + "topic";
public static final String MESSAGE_KEY = PREFIX + "messageKey";
public static final String PARTITION_ID = PREFIX + "partitionId";
public static final String OFFSET = PREFIX + "offset";
public static final String NEXT_OFFSET = PREFIX + "nextOffset";
public static final String ACKNOWLEDGMENT = PREFIX + "acknowledgment";
}

215
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaProducerContext.java
View File

@@ -1,215 +0,0 @@
/*
* Copyright 2013-2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.springframework.beans.factory.BeanNameAware;
import org.springframework.context.SmartLifecycle;
import org.springframework.integration.support.context.NamedComponent;
import org.springframework.util.StringUtils;
/**
* @author Soby Chacko
* @author Rajasekar Elango
* @author Ilayaperumal Gopinathan
* @author Gary Russell
* @author Artem Bilan
* @author Marius Bogoevici
* @since 0.5
*/
public class KafkaProducerContext implements SmartLifecycle, NamedComponent, BeanNameAware {
private static final Log logger = LogFactory.getLog(KafkaProducerContext.class);
private final AtomicBoolean running = new AtomicBoolean();
private volatile Map<String, ProducerConfiguration<?, ?>> producerConfigurations;
private volatile ProducerConfiguration<?, ?> theProducerConfiguration;
private String beanName;
private int phase = 0;
private boolean autoStartup = true;
public ProducerConfiguration<?, ?> getTopicConfiguration(final String topic) {
if (this.theProducerConfiguration != null) {
if (topic.matches(this.theProducerConfiguration.getProducerMetadata().getTopic())) {
return this.theProducerConfiguration;
}
}
Collection<ProducerConfiguration<?, ?>> topics = this.producerConfigurations.values();
for (final ProducerConfiguration<?, ?> producerConfiguration : topics) {
if (topic.matches(producerConfiguration.getProducerMetadata().getTopic())) {
return producerConfiguration;
}
}
return null;
}
public Map<String, ProducerConfiguration<?, ?>> getProducerConfigurations() {
return this.producerConfigurations;
}
public void setProducerConfigurations(Map<String, ProducerConfiguration<?, ?>> producerConfigurations) {
this.producerConfigurations = producerConfigurations;
if (this.producerConfigurations.size() == 1) {
this.theProducerConfiguration = this.producerConfigurations.values().iterator().next();
}
}
/**
* @return the component type.
* @since 1.0
*/
@Override
public String getComponentType() {
return "kafka:producer-context";
}
/**
* @param name the bean name.
* @since 1.0
*/
@Override
public void setBeanName(String name) {
this.beanName = name;
}
/**
* @param phase the phase to set.
* @see SmartLifecycle
* @since 1.0
*/
public void setPhase(int phase) {
this.phase = phase;
}
/**
* @param autoStartup the autoStartup to set.
* @see SmartLifecycle
* @since 1.0
*/
public void setAutoStartup(boolean autoStartup) {
this.autoStartup = autoStartup;
}
/**
* @return the component name.
* @since 1.0
*/
@Override
public String getComponentName() {
return this.beanName;
}
protected void doStart() {
}
protected void doStop() {
if (this.producerConfigurations != null) {
for (ProducerConfiguration<?, ?> producerConfiguration : this.producerConfigurations.values()) {
producerConfiguration.stop();
}
}
}
@Override
public final void start() {
if (this.running.compareAndSet(false, true)) {
doStart();
}
else {
if (logger.isDebugEnabled()) {
logger.debug(getComponentType() + ":" + getComponentName() + " is already running");
}
}
}
@Override
public final void stop() {
if (this.running.compareAndSet(true, false)) {
doStop();
}
else {
if (logger.isDebugEnabled()) {
logger.debug(getComponentType() + ":" + getComponentName() + " is not running");
}
}
}
@Override
public boolean isRunning() {
return this.running.get();
}
@Override
public int getPhase() {
return this.phase;
}
@Override
public boolean isAutoStartup() {
return this.autoStartup;
}
@Override
public void stop(Runnable callback) {
stop();
callback.run();
}
public Future<RecordMetadata> send(String topic, Object messageKey, Object messagePayload) {
return send(topic, null, messageKey, messagePayload);
}
public Future<RecordMetadata> send(String topic, Integer partition, Object messageKey, Object messagePayload) {
if (!this.running.get()) {
start();
}
// only try to look up for a producer configuration if the topic is passed as argument
// if no topic is configured, then we'll fall back to the default if a single
// producer configuration is available
ProducerConfiguration<?, ?> producerConfiguration =
StringUtils.hasText(topic) ? getTopicConfiguration(topic) : null;
if (producerConfiguration != null) {
return producerConfiguration.convertAndSend(topic, partition, messageKey, messagePayload);
}
// if there is a single producer configuration then use that config to send message.
else if (this.theProducerConfiguration != null) {
return this.theProducerConfiguration.convertAndSend(topic, partition, messageKey, messagePayload);
}
else {
throw new IllegalStateException("Could not send messages as there are multiple producer configurations " +
"with no topic information found from the message header.");
}
}
}

83
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/LoggingProducerListener.java
View File

@@ -1,83 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.util.ObjectUtils;
/**
* {@link ProducerListener} that logs exceptions thrown when sending messages.
*
* @author Marius Bogoevici
* @since 1.3
*/
public class LoggingProducerListener extends DefaultProducerListener {
private static final Log log = LogFactory.getLog(LoggingProducerListener.class);
private boolean includeContents = true;
private int maxContentLogged = 100;
/**
* Whether the log message should include the contents (key and payload).
*
* @param includeContents true if the contents of the message should be logged
*/
public void setIncludeContents(boolean includeContents) {
this.includeContents = includeContents;
}
/**
* The maximum amount of data to be logged for either key or password. As message sizes may vary and
* become fairly large, this allows limiting the amount of data sent to logs.
*
* @param maxContentLogged the maximum amount of data being logged.
*/
public void setMaxContentLogged(int maxContentLogged) {
this.maxContentLogged = maxContentLogged;
}
@Override
public void onError(String topic, Integer partition, Object key, Object payload, Exception exception) {
if (log.isErrorEnabled()) {
StringBuffer logOutput = new StringBuffer();
logOutput.append("Exception thrown when sending a message");
if (includeContents) {
logOutput.append(" with key='"
+ toDisplayString(ObjectUtils.nullSafeToString(key), maxContentLogged) + "'");
logOutput.append(" and payload='"
+ toDisplayString(ObjectUtils.nullSafeToString(payload),maxContentLogged) + "'");
}
logOutput.append(" to topic " + topic);
if (partition != null) {
logOutput.append(" and partition " +partition);
}
logOutput.append(":");
log.error(logOutput, exception);
}
}
private String toDisplayString(String original, int maxCharacters) {
if (original.length() <= maxCharacters) {
return original;
}
return original.substring(0, maxCharacters) + "...";
}
}

46
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/MessageLeftOverTracker.java
View File

@@ -1,46 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import java.util.ArrayList;
import java.util.List;
import kafka.message.MessageAndMetadata;
/**
* @author Soby Chacko
* @since 0.5
*/
@Deprecated
public class MessageLeftOverTracker<K,V> {
private final List<MessageAndMetadata<K,V>> messageLeftOverFromPreviousPoll = new ArrayList<MessageAndMetadata<K,V>>();
public void addMessageAndMetadata(final MessageAndMetadata<K,V> messageAndMetadata){
messageLeftOverFromPreviousPoll.add(messageAndMetadata);
}
public List<MessageAndMetadata<K,V>> getMessageLeftOverFromPreviousPoll(){
return messageLeftOverFromPreviousPoll;
}
public void clearMessagesLeftOver(){
messageLeftOverFromPreviousPoll.clear();
}
public int getCurrentCount() {
return messageLeftOverFromPreviousPoll.size();
}
}

201
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerConfiguration.java
View File

@@ -1,201 +0,0 @@
/*
* Copyright 2013-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.kafka.common.KafkaException;
import org.springframework.core.convert.ConversionService;
import org.springframework.core.convert.TypeDescriptor;
import org.springframework.core.convert.converter.GenericConverter;
import org.springframework.core.convert.support.GenericConversionService;
import org.springframework.core.serializer.support.SerializingConverter;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
/**
* @author Soby Chacko
* @author Rajasekar Elango
* @author Ilayaperumal Gopinathan
* @author Gary Russell
* @author Marius Bogoevici
* @author Artem Bilan
* @author Martin Dam
* @since 0.5
*/
public class ProducerConfiguration<K, V> {
private final Producer<K, V> producer;
private final ProducerMetadata<K, V> producerMetadata;
private ConversionService conversionService;
private ProducerListener producerListener;
public ProducerConfiguration(ProducerMetadata<K, V> producerMetadata, Producer<K, V> producer) {
Assert.notNull(producerMetadata);
Assert.notNull(producer);
this.producerMetadata = producerMetadata;
this.producer = producer;
GenericConversionService genericConversionService = new GenericConversionService();
genericConversionService.addConverter(new StringBytesConverter());
genericConversionService.addConverter(Object.class, byte[].class, new SerializingConverter());
this.conversionService = genericConversionService;
}
public void setConversionService(ConversionService conversionService) {
Assert.notNull(conversionService, "Conversion service must not be null");
this.conversionService = conversionService;
}
public void setProducerListener(ProducerListener producerListener) {
this.producerListener = producerListener;
}
public ProducerMetadata<K, V> getProducerMetadata() {
return this.producerMetadata;
}
public Future<RecordMetadata> send(String topic, K messageKey, V messagePayload) {
if (this.getProducerMetadata().getPartitioner() != null) {
String targetTopic = StringUtils.hasText(topic) ? topic : this.producerMetadata.getTopic();
int partition = this.getProducerMetadata().getPartitioner().partition(messageKey,
this.producer.partitionsFor(targetTopic).size());
return this.send(targetTopic, partition, messageKey, messagePayload);
}
return this.send(topic, null, messageKey, messagePayload);
}
public Future<RecordMetadata> send(String topic, Integer partition, K messageKey, V messagePayload) {
String targetTopic = StringUtils.hasText(topic) ? topic : this.producerMetadata.getTopic();
//If partition is sent by producer context then that takes precedence over custom partitioner.
if (partition == null && this.getProducerMetadata().getPartitioner() != null) {
partition = this.getProducerMetadata().getPartitioner().partition(messageKey,
this.producer.partitionsFor(targetTopic).size());
}
ProducerRecord<K, V> record = new ProducerRecord<>(targetTopic, partition, messageKey, messagePayload);
Future<RecordMetadata> future;
if (producerListener == null) {
future = this.producer.send(record);
}
else {
ProducerListenerInvokingCallback callback =
new ProducerListenerInvokingCallback(targetTopic, partition, messageKey, messagePayload,
producerListener);
future = this.producer.send(record, callback);
}
if (!producerMetadata.isSync()) {
return future;
}
else {
try {
if (producerMetadata.getSendTimeout() <= 0) {
future.get();
}
else {
future.get(producerMetadata.getSendTimeout(), TimeUnit.MILLISECONDS);
}
}
catch (InterruptedException | ExecutionException | TimeoutException e) {
throw new KafkaException(e);
}
return future;
}
}
public Future<RecordMetadata> convertAndSend(String topic, Integer partition, Object messageKey,
Object messagePayload) {
return send(topic, partition, convertKeyIfNecessary(messageKey), convertPayloadIfNecessary(messagePayload));
}
public Future<RecordMetadata> convertAndSend(String topic, Object messageKey, Object messagePayload) {
return send(topic, convertKeyIfNecessary(messageKey), convertPayloadIfNecessary(messagePayload));
}
private K convertKeyIfNecessary(Object messageKey) {
if (messageKey != null) {
if (getProducerMetadata().getKeyClassType().isAssignableFrom(
messageKey.getClass())) {
return getProducerMetadata().getKeyClassType().cast(messageKey);
}
return this.conversionService.convert(messageKey, this.producerMetadata.getKeyClassType());
}
else {
return null;
}
}
private V convertPayloadIfNecessary(Object messagePayload) {
if (messagePayload != null) {
if (getProducerMetadata().getValueClassType().isAssignableFrom(
messagePayload.getClass())) {
return getProducerMetadata().getValueClassType().cast(messagePayload);
}
return this.conversionService.convert(messagePayload, this.producerMetadata.getValueClassType());
}
else {
return null;
}
}
public void stop() {
this.producer.close();
}
@Override
public String toString() {
return "ProducerConfiguration{" +
"producer=" + this.producer +
", producerMetadata=" + this.producerMetadata +
", conversionService=" + this.conversionService +
'}';
}
private class StringBytesConverter implements GenericConverter {
@Override
public Set<ConvertiblePair> getConvertibleTypes() {
Set<ConvertiblePair> convertiblePairs = new HashSet<ConvertiblePair>();
convertiblePairs.add(new ConvertiblePair(String.class, byte[].class));
convertiblePairs.add(new ConvertiblePair(byte[].class, String.class));
return convertiblePairs;
}
@Override
public Object convert(Object source, TypeDescriptor sourceType, TypeDescriptor targetType) {
if (source instanceof String) {
return ((String) source).getBytes(ProducerConfiguration.this.producerMetadata.getCharset());
}
else {
return new String((byte[]) source, ProducerConfiguration.this.producerMetadata.getCharset());
}
}
}
}

79
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerFactoryBean.java
View File

@@ -1,79 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.springframework.beans.factory.FactoryBean;
/**
* @author Soby Chacko
* @author Marius Bogoevici
* @since 0.5
*/
public class ProducerFactoryBean<K, V> implements FactoryBean<Producer<K, V>> {
private static final Log LOGGER = LogFactory.getLog(ProducerFactoryBean.class);
private final String brokerList;
private final ProducerMetadata<K, V> producerMetadata;
private Properties producerProperties = new Properties();
public ProducerFactoryBean(final ProducerMetadata<K, V> producerMetadata, final String brokerList,
final Properties producerProperties) {
this.producerMetadata = producerMetadata;
this.brokerList = brokerList;
if (producerProperties != null) {
this.producerProperties = producerProperties;
}
}
public ProducerFactoryBean(final ProducerMetadata<K, V> producerMetadata, final String brokerList) {
this(producerMetadata, brokerList, null);
}
@Override
public Producer<K, V> getObject() throws Exception {
final Properties props = new Properties();
props.putAll(producerProperties);
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList);
props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, producerMetadata.getCompressionType().name());
props.put(ProducerConfig.BATCH_SIZE_CONFIG, producerMetadata.getBatchBytes());
LOGGER.info("Using producer properties => " + props);
return new KafkaProducer<>(props,
producerMetadata.getKeySerializer(),
producerMetadata.getValueSerializer());
}
@Override
public Class<?> getObjectType() {
return Producer.class;
}
@Override
public boolean isSingleton() {
return true;
}
}

55
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerListener.java
View File

@@ -1,55 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import org.apache.kafka.clients.producer.RecordMetadata;
/**
* Listener for handling outbound Kafka messages. Exactly one of its methods will be invoked, depending on whether
* the write has been acknowledged or not.
*
* Its main goal is to provide a stateless singleton delegate for {@link org.apache.kafka.clients.producer.Callback}s,
* which, in all but the most trivial cases, requires creating a separate instance per message.
*
* @see org.apache.kafka.clients.producer.Callback
* @author Marius Bogoevici
* @since 1.3
*/
public interface ProducerListener {
/**
* Invoked after the successful send of a message (that is, after it has been acknowledged by the broker)
*
* @param topic the destination topic
* @param partition the destination partition (could be null)
* @param key the key of the outbound message
* @param value the payload of the outbound message
* @param recordMetadata the result of the successful send operation
*/
void onSuccess(String topic, Integer partition, Object key, Object value, RecordMetadata recordMetadata);
/**
* Invoked after an attempt to send a message has failed
*
* @param topic the destination topic
* @param partition the destination partition (could be null)
* @param key the key of the outbound message
* @param value the payload of the outbound message
* @param exception the exception thrown
*/
void onError(String topic, Integer partition, Object key, Object value,Exception exception);
}

61
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerListenerInvokingCallback.java
View File

@@ -1,61 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.springframework.util.Assert;
/**
* Adapts the {@link org.apache.kafka.clients.producer.Callback} interface of the
* {@link org.apache.kafka.clients.producer.Producer} to a {@link ProducerListener}.
*
* @author Marius Bogoevici
*/
public class ProducerListenerInvokingCallback implements Callback {
private final String topic;
private final Integer partition;
private final Object key;
private final Object payload;
private final ProducerListener producerListener;
public ProducerListenerInvokingCallback(String topic, Integer partition, Object key, Object payload,
ProducerListener producerListener) {
Assert.notNull(producerListener, "must not be null");
this.topic = topic;
this.partition = partition;
this.key = key;
this.payload = payload;
this.producerListener = producerListener;
}
@Override
public void onCompletion(RecordMetadata metadata, Exception exception) {
if (exception != null) {
producerListener.onError(topic,partition,key, payload,exception);
}
else {
producerListener.onSuccess(topic, partition, key, payload, metadata);
}
}
}

173
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerMetadata.java
View File

@@ -1,173 +0,0 @@
/*
* Copyright 2002-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import java.nio.charset.Charset;
import org.apache.kafka.common.serialization.Serializer;
import org.springframework.util.Assert;
import kafka.producer.Partitioner;
/**
*
* @author Soby Chacko
* @author Rajasekar Elango
* @author Marius Bogoevici
* @author Artem Bilan
*
* @since 0.5
*/
public class ProducerMetadata<K,V> {
private final Class<K> keyClassType;
private final Class<V> valueClassType;
private Serializer<K> keySerializer;
private Serializer<V> valueSerializer;
private final String topic;
private Partitioner partitioner;
private CompressionType compressionType = CompressionType.none;
private int batchBytes = 16384;
private int sendTimeout = 0;
private boolean sync = false;
private Charset charset = Charset.forName("UTF8");
public ProducerMetadata(final String topic, Class<K> keyClassType, Class<V> valueClassType,
Serializer<K> keySerializer, Serializer<V> valueSerializer) {
Assert.notNull(topic, "Topic cannot be null");
Assert.notNull(keyClassType, "Key class type serializer cannot be null");
Assert.notNull(valueClassType, "Value class type cannot be null");
Assert.notNull(keySerializer, "Value serializer cannot be null");
Assert.notNull(valueSerializer, "Value serializer cannot be null");
this.topic = topic;
this.keyClassType = keyClassType;
this.valueClassType = valueClassType;
this.keySerializer = keySerializer;
this.valueSerializer = valueSerializer;
}
public String getTopic() {
return topic;
}
public Serializer<K> getKeySerializer() {
return keySerializer;
}
public Serializer<V> getValueSerializer() {
return valueSerializer;
}
public CompressionType getCompressionType() {
return compressionType;
}
public void setCompressionType(CompressionType compressionType) {
Assert.notNull(compressionType, "Compression type cannot be null");
this.compressionType = compressionType;
}
public int getBatchBytes() {
return batchBytes;
}
public void setBatchBytes(int batchBytes) {
Assert.isTrue(batchBytes > 0, "Buffer size must be greater than zero");
this.batchBytes = batchBytes;
}
public Partitioner getPartitioner() {
return partitioner;
}
public void setPartitioner(Partitioner partitioner) {
this.partitioner = partitioner;
}
public Class<K> getKeyClassType() {
return keyClassType;
}
public Class<V> getValueClassType() {
return valueClassType;
}
public int getSendTimeout() {
return sendTimeout;
}
public void setSendTimeout(int sendTimeout) {
this.sendTimeout = sendTimeout;
}
public boolean isSync() {
return sync;
}
public void setSync(boolean sync) {
this.sync = sync;
}
public Charset getCharset() {
return charset;
}
/**
* The character encoding to preform {@code String <-> byte[]} conversion
* instead of general (de)serialization.
* @param charset the charset encoding to use.
* @since 1.3
*/
public void setCharset(Charset charset) {
this.charset = charset;
}
@Override
public String toString() {
return "ProducerMetadata{" +
"keyClassType=" + this.keyClassType +
", valueClassType=" + this.valueClassType +
", keySerializer=" + this.keySerializer +
", valueSerializer=" + this.valueSerializer +
", topic='" + this.topic + '\'' +
", partitioner=" + this.partitioner +
", compressionType=" + this.compressionType +
", batchBytes=" + this.batchBytes +
", sync=" + this.sync +
", sendTimeout=" + this.sendTimeout +
", charset=" + this.charset +
'}';
}
public enum CompressionType {
none,
gzip,
snappy
}
}

58
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/TopicFilterConfiguration.java
View File

@@ -1,58 +0,0 @@
/*
* Copyright 2002-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import kafka.consumer.Blacklist;
import kafka.consumer.TopicFilter;
import kafka.consumer.Whitelist;
/**
* @author Rajasekar Elango
* @author Artem Bilan
* @since 0.5
*/
@Deprecated
public class TopicFilterConfiguration {
private final int numberOfStreams;
private final TopicFilter topicFilter;
public TopicFilterConfiguration(final String pattern, final int numberOfStreams, final boolean exclude) {
this.numberOfStreams = numberOfStreams;
if (exclude) {
this.topicFilter = new Blacklist(pattern);
}
else {
this.topicFilter = new Whitelist(pattern);
}
}
public TopicFilter getTopicFilter() {
return this.topicFilter;
}
public int getNumberOfStreams() {
return this.numberOfStreams;
}
@Override
public String toString() {
return this.topicFilter.toString() + " : " + this.numberOfStreams;
}
}

72
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ZookeeperConnect.java
View File

@@ -1,72 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import org.springframework.integration.kafka.core.ZookeeperConnectDefaults;
/**
* @author Soby Chacko
* @since 0.5
*/
public class ZookeeperConnect {
private String zkConnect = ZookeeperConnectDefaults.ZK_CONNECT;
private String zkConnectionTimeout = ZookeeperConnectDefaults.ZK_CONNECTION_TIMEOUT;
private String zkSessionTimeout = ZookeeperConnectDefaults.ZK_SESSION_TIMEOUT;
private String zkSyncTime = ZookeeperConnectDefaults.ZK_SYNC_TIME;
public ZookeeperConnect() {
}
public ZookeeperConnect(String zkConnect) {
this.zkConnect = zkConnect;
}
public String getZkConnect() {
return zkConnect;
}
public void setZkConnect(final String zkConnect) {
this.zkConnect = zkConnect;
}
public String getZkConnectionTimeout() {
return zkConnectionTimeout;
}
public void setZkConnectionTimeout(final String zkConnectionTimeout) {
this.zkConnectionTimeout = zkConnectionTimeout;
}
public String getZkSessionTimeout() {
return zkSessionTimeout;
}
public void setZkSessionTimeout(final String zkSessionTimeout) {
this.zkSessionTimeout = zkSessionTimeout;
}
public String getZkSyncTime() {
return zkSyncTime;
}
public void setZkSyncTime(final String zkSyncTime) {
this.zkSyncTime = zkSyncTime;
}
}

4
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/package-info.java
View File

@@ -1,4 +0,0 @@
/**
* Provides various support classes used across Spring Integration Kafka Components.
*/
package org.springframework.integration.kafka.support;

57
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/EncoderAdaptingSerializer.java
View File

@@ -1,57 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.util;
import java.util.Map;
import kafka.serializer.DefaultEncoder;
import kafka.serializer.Encoder;
import org.apache.kafka.common.serialization.Serializer;
/**
* An adapter from the pre-0.8.2 Kafka {@link Encoder} to the {@link Serializer} interface used
* by the new client.
*
* @author Marius Bogoevici
*/
public class EncoderAdaptingSerializer<T> implements Serializer<T> {
private final Encoder<T> encoder;
public EncoderAdaptingSerializer(Encoder<T> encoder) {
this.encoder = encoder;
}
public Encoder<T> getEncoder() {
return encoder;
}
@Override
public void configure(Map<String, ?> configs, boolean isKey) {
// no-op
}
@Override
public byte[] serialize(String topic, T data) {
return encoder.toBytes(data);
}
@Override
public void close() {
// no-op;
}
}

45
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/LoggingUtils.java
View File

@@ -1,45 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.util;
/**
* Utilities for logging data
*
* @author Marius Bogoevici
*/
public class LoggingUtils {
public static String asCommaSeparatedHexDump(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return "[]";
}
else if (bytes.length == 1) {
return String.format("[%s]", Integer.toHexString(bytes[0]));
}
else {
StringBuilder buffer = new StringBuilder("[");
for (int i = 0; i < bytes.length - 1; i++) {
buffer.append(Integer.toHexString(bytes[i]));
buffer.append(",");
}
buffer.append(Integer.toHexString(bytes[bytes.length]));
buffer.append("]");
return buffer.toString();
}
}
}

43
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/MessageUtils.java
View File

@@ -1,43 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.util;
import org.springframework.integration.kafka.core.KafkaMessage;
import kafka.serializer.Decoder;
import kafka.utils.Utils$;
/**
* @author Marius Bogoevici
*/
public class MessageUtils {
public static <T> T decodeKey(KafkaMessage message, Decoder<T> decoder) {
if (!message.getMessage().hasKey()) {
return null;
}
return decoder.fromBytes(Utils$.MODULE$.readBytes(message.getMessage().key()));
}
public static <T> T decodePayload(KafkaMessage message, Decoder<T> decoder) {
if (message.getMessage().isNull()) {
return null;
}
return decoder.fromBytes(Utils$.MODULE$.readBytes(message.getMessage().payload()));
}
}

160
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/TopicUtils.java
View File

@@ -1,160 +0,0 @@
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.util;
import java.util.List;
import java.util.Properties;
import kafka.common.LeaderNotAvailableException;
import org.I0Itec.zkclient.ZkClient;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.integration.kafka.core.TopicNotFoundException;
import org.springframework.retry.RetryCallback;
import org.springframework.retry.RetryContext;
import org.springframework.retry.RetryPolicy;
import org.springframework.retry.backoff.ExponentialBackOffPolicy;
import org.springframework.retry.policy.CompositeRetryPolicy;
import org.springframework.retry.policy.SimpleRetryPolicy;
import org.springframework.retry.policy.TimeoutRetryPolicy;
import org.springframework.retry.support.RetryTemplate;
import kafka.admin.AdminUtils;
import kafka.api.TopicMetadata;
import kafka.common.ErrorMapping;
import kafka.javaapi.PartitionMetadata;
import kafka.utils.ZKStringSerializer$;
import kafka.utils.ZkUtils;
import scala.collection.Map;
import scala.collection.Seq;
/**
* Utilities for interacting with Kafka topics
*
* @author Marius Bogoevici
*/
public class TopicUtils {
private static Log log = LogFactory.getLog(TopicUtils.class);
public static final int METADATA_VERIFICATION_TIMEOUT = 5000;
public static final int METADATA_VERIFICATION_RETRY_ATTEMPTS = 10;
public static final double METADATA_VERIFICATION_RETRY_BACKOFF_MULTIPLIER = 1.5;
public static final int METADATA_VERIFICATION_RETRY_INITIAL_INTERVAL = 100;
public static final int METADATA_VERIFICATION_MAX_INTERVAL = 1000;
/**
* Creates a topic in Kafka or validates that it exists with the requested number of partitions,
* and returns only after the topic has been fully created
* @param zkAddress the address of the Kafka ZooKeeper instance
* @param topicName the name of the topic
* @param numPartitions the number of partitions
* @param replicationFactor the replication factor
* @return {@link TopicMetadata} information for the topic
*/
public static TopicMetadata ensureTopicCreated(final String zkAddress, final String topicName,
final int numPartitions, int replicationFactor) {
final int sessionTimeoutMs = 10000;
final int connectionTimeoutMs = 10000;
final ZkClient zkClient = new ZkClient(zkAddress, sessionTimeoutMs, connectionTimeoutMs,
ZKStringSerializer$.MODULE$);
try {
// The following is basically copy/paste from AdminUtils.createTopic() with
// createOrUpdateTopicPartitionAssignmentPathInZK(..., update=true)
Properties topicConfig = new Properties();
Seq<Object> brokerList = ZkUtils.getSortedBrokerList(zkClient);
scala.collection.Map<Object, Seq<Object>> replicaAssignment =
AdminUtils.assignReplicasToBrokers(brokerList, numPartitions, replicationFactor, -1, -1);
return ensureTopicCreated(zkClient, topicName, numPartitions, topicConfig, replicaAssignment);
}
finally {
zkClient.close();
}
}
/**
* Creates a topic in Kafka and returns only after the topic has been fully and an produce metadata.
* @param zkClient an open {@link ZkClient} connection to Zookeeper
* @param topicName the name of the topic
* @param numPartitions the number of partitions for the topic
* @param topicConfig additional topic configuration properties
* @param replicaAssignment the mapping of partitions to broker
* @return {@link TopicMetadata} information for the topic
*/
public static TopicMetadata ensureTopicCreated(final ZkClient zkClient, final String topicName,
final int numPartitions, Properties topicConfig, Map<Object, Seq<Object>> replicaAssignment) {
AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK(zkClient, topicName, replicaAssignment, topicConfig,
true);
RetryTemplate retryTemplate = new RetryTemplate();
CompositeRetryPolicy policy = new CompositeRetryPolicy();
TimeoutRetryPolicy timeoutRetryPolicy = new TimeoutRetryPolicy();
timeoutRetryPolicy.setTimeout(METADATA_VERIFICATION_TIMEOUT);
SimpleRetryPolicy simpleRetryPolicy = new SimpleRetryPolicy();
simpleRetryPolicy.setMaxAttempts(METADATA_VERIFICATION_RETRY_ATTEMPTS);
policy.setPolicies(new RetryPolicy[] {timeoutRetryPolicy, simpleRetryPolicy});
retryTemplate.setRetryPolicy(policy);
ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
backOffPolicy.setInitialInterval(METADATA_VERIFICATION_RETRY_INITIAL_INTERVAL);
backOffPolicy.setMultiplier(METADATA_VERIFICATION_RETRY_BACKOFF_MULTIPLIER);
backOffPolicy.setMaxInterval(METADATA_VERIFICATION_MAX_INTERVAL);
retryTemplate.setBackOffPolicy(backOffPolicy);
try {
return retryTemplate.execute(new RetryCallback<TopicMetadata, Exception>() {
@Override
public TopicMetadata doWithRetry(RetryContext context) throws Exception {
TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topicName, zkClient);
if (topicMetadata.errorCode() != ErrorMapping.NoError() ||
!topicName.equals(topicMetadata.topic())) {
// downcast to Exception because that's what the error throws
throw (Exception) ErrorMapping.exceptionFor(topicMetadata.errorCode());
}
List<PartitionMetadata> partitionMetadatas =
new kafka.javaapi.TopicMetadata(topicMetadata).partitionsMetadata();
if (partitionMetadatas.size() != numPartitions) {
throw new IllegalStateException("The number of expected partitions was: " +
numPartitions + ", but " + partitionMetadatas.size() + " have been found instead");
}
for (PartitionMetadata partitionMetadata : partitionMetadatas) {
if (partitionMetadata.errorCode() != ErrorMapping.NoError()) {
throw (Exception) ErrorMapping.exceptionFor(partitionMetadata.errorCode());
}
if (partitionMetadata.leader() == null) {
throw new LeaderNotAvailableException();
}
}
return topicMetadata;
}
});
}
catch (Exception e) {
log.error(String.format("Cannot retrieve metadata for topic '%s'", topicName), e);
throw new TopicNotFoundException(topicName);
}
}
}

639
spring-integration-kafka/src/main/resources/org/springframework/integration/kafka/config/spring-integration-kafka-2.0.xsd
View File

@@ -14,479 +14,15 @@
<xsd:annotation>
<xsd:documentation><![CDATA[
Defines the configuration elements for the Spring Integration
Kafka Adapter.
Kafka Adapters.
]]></xsd:documentation>
</xsd:annotation>
<xsd:element name="zookeeper-connect">
<xsd:annotation>
<xsd:documentation><![CDATA[
Defines a Kafka server information.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:attribute name="id" type="xsd:string" use="required"/>
<xsd:attribute name="zk-connect" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the Kafka server URL
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="zk-connection-timeout" use="optional">
<xsd:annotation>
<xsd:documentation><![CDATA[
Specifies the max time in ms that the client waits to establish a connection to zookeeper.
]]></xsd:documentation>
</xsd:annotation>
<xsd:simpleType>
<xsd:union memberTypes="xsd:int xsd:string"/>
</xsd:simpleType>
</xsd:attribute>
<xsd:attribute name="zk-session-timeout" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[
The zookeeper session timeout.
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="zk-sync-time" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[
Max time for how far a zookeeper follower can be behind a zookeeper leader in ms.
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="auto-commit-interval" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[
The frequency that the consumed offsets are committed to zookeeper in ms.
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
</xsd:complexType>
</xsd:element>
<xsd:element name="producer-context">
<xsd:annotation>
<xsd:documentation><![CDATA[
Defines a producer context.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:sequence>
<xsd:element name="producer-configurations" minOccurs="0" maxOccurs="1">
<xsd:annotation>
<xsd:documentation><![CDATA[
Groups kafka topic configurations.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:choice>
<xsd:element name="producer-configuration" maxOccurs="unbounded">
<xsd:annotation>
<xsd:documentation><![CDATA[
Declares a kafka topic configuration which drives how and where a topic is sent to broker/s.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:attribute name="topic" type="xsd:string" use="required">
<xsd:annotation>
<xsd:documentation>
The topic configured by this configuration.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="broker-list" use="required" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[
list of comma separated kafka brokers.
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="key-class-type" use="optional" type="xsd:string" default="[B">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
Class type used for the key
</xsd:documentation>
<tool:annotation kind="direct">
<tool:expected-type type="java.lang.Class"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="value-class-type" use="optional" type="xsd:string" default="[B">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
Class type used for the value
</xsd:documentation>
<tool:annotation kind="direct">
<tool:expected-type type="java.lang.Class"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="value-encoder" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
[DEPRECATED]
Custom implementation of a Kafka Encoder for encoding message values.
This option is deprecated, 'value-serializer' is the recommended option.
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="kafka.serializer.Encoder"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="value-serializer" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
Custom implementation of a Kafka Serializer for message values.
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="org.apache.kafka.common.serialization.Serializer"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="key-encoder" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
[DEPRECATED]
Custom implementation of a Kafka Encoder for encoding message keys.
This option is deprecated, 'key-serializer' is the recommended option.
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="kafka.serializer.Encoder"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="key-serializer" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
Custom implementation of a Kafka Serializer for message keys.
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="kafka.serializer.Encoder"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="conversion-service" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
Conversion service for transforming inbound objects into the target key and value
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.core.convert.ConversionService"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="producer-listener" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
Listener for notifying when a message has been sent or has failed.
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.integration.kafka.support.ProducerListener"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="compression-type" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the type of compression codec used for message compression.
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="partitioner" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
[DEPRECATED]
Custom Kafka key partitioner.
Deprecated in favor of 'partition-id' ('partition-id-expression')
on the 'outbound-channel-adapter'.
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="kafka.producer.Partitioner"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="batch-bytes" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
number of messages to batch at this producer.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="sync" use="optional" type="xsd:boolean">
<xsd:annotation>
<xsd:documentation>
If true, sends messages synchronously. Asynchronously (default) otherwise
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="send-timeout" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
If sync=true, specify timeout in milliseconds. 0 or below indicate forever (default)
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="charset" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
The character encoding to preform String to/from byte[] conversion
instead of general (de)serialization. Defaults to UTF8.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
</xsd:complexType>
</xsd:element>
</xsd:choice>
</xsd:complexType>
</xsd:element>
</xsd:sequence>
<xsd:attribute name="id" type="xsd:string" use="required"/>
<xsd:attribute name="producer-properties" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
Kafka producer properties to use for all producers
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="java.util.Properties"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attributeGroup ref="integration:smartLifeCycleAttributeGroup" />
</xsd:complexType>
</xsd:element>
<xsd:element name="consumer-context">
<xsd:annotation>
<xsd:documentation><![CDATA[
[DEPRECATED]
Defines a producer context.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:sequence>
<xsd:element name="consumer-configurations" minOccurs="0" maxOccurs="1">
<xsd:annotation>
<xsd:documentation><![CDATA[
Groups kafka topic configurations.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:choice>
<xsd:element name="consumer-configuration" maxOccurs="unbounded">
<xsd:annotation>
<xsd:documentation><![CDATA[
Declares a kafka topic configuration which drives how and where a topic is sent to broker/s.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:choice>
<xsd:element name="topic" maxOccurs="unbounded">
<xsd:complexType>
<xsd:attribute name="id" type="xsd:string" use="required"/>
<xsd:attribute name="streams" type="xsd:string" use="required"/>
</xsd:complexType>
</xsd:element>
<xsd:element name="topic-filter" maxOccurs="1">
<xsd:complexType>
<xsd:attribute name="pattern" type="xsd:string" use="required">
<xsd:annotation>
<xsd:documentation>
Regex pattern to match topic
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="streams" type="xsd:string" use="required">
<xsd:annotation>
<xsd:documentation><![CDATA[
Number of streams (threads) to use to consume messages
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="exclude" use="optional" default="false">
<xsd:annotation>
<xsd:documentation><![CDATA[
If exclude is false, it uses whitelist to include topics matching given pattern.
If exclude is true, it uses blacklist to exclude topics matching given pattern.
Default value is false.
]]></xsd:documentation>
</xsd:annotation>
<xsd:simpleType>
<xsd:union memberTypes="xsd:boolean xsd:string"/>
</xsd:simpleType>
</xsd:attribute>
</xsd:complexType>
</xsd:element>
</xsd:choice>
<xsd:attribute name="max-messages" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates max messages to aggregate in a single call to receive
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="group-id" use="required" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the Kafka consumer group id
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="value-decoder" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
Custom implementation of a Kafka Decoder for values.
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="kafka.serializer.Decoder"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="key-decoder" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
Custom implementation of a Kafka Decoder for keys.
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="kafka.serializer.Decoder"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="executor" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
A java.util.concurrent.Executor bean reference.
Typically
org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor.
Used to execute iterator task on KafkaStreams.
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="java.util.concurrent.Executor"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
</xsd:complexType>
</xsd:element>
</xsd:choice>
</xsd:complexType>
</xsd:element>
</xsd:sequence>
<xsd:attribute name="id" type="xsd:string" use="required"/>
<xsd:attribute name="consumer-timeout" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the Kafka consumer timeout ms
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="zookeeper-connect" use="required" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
Kafka Server Bean Name
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.integration.kafka.support.ZookeeperConnect"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="consumer-properties" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
Kafka consumer properties to use for all consumers
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="java.util.Properties"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
</xsd:complexType>
</xsd:element>
<xsd:element name="inbound-channel-adapter">
<xsd:annotation>
<xsd:documentation>
[DEPRECATED]
The definition for the Spring Integration Kafka
Inbound Channel Adapter.
This High Level Consumer Adapter is deprecated in favor of
message-driven-channel-adapter - based on the Simple Consumer API.
</xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:sequence>
<xsd:element ref="integration:poller" minOccurs="0" maxOccurs="1"/>
</xsd:sequence>
<xsd:attributeGroup ref="integration:channelAdapterAttributes"/>
<xsd:attribute name="send-timeout" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Maximum amount of time in milliseconds to wait when sending a message to the channel
if such channel may block. For example, a Queue Channel can block until space is available
if its maximum capacity has been reached.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="group-id" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the Kafka consumer group id
]]></xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="kafka-consumer-context-ref" use="required" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<xsd:documentation>
Kafka consumer context reference.
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.integration.kafka.support.KafkaConsumerContext"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
</xsd:complexType>
</xsd:element>
<xsd:element name="outbound-channel-adapter">
<xsd:annotation>
<xsd:documentation>
Defines kafka outbound channel adapter that writes the contents of the
Message to kafka broker.
Defines the Consumer Endpoint for the KafkaProducerMessageHandler
that writes the contents of the Message to kafka broker.
</xsd:documentation>
</xsd:annotation>
@@ -497,14 +33,14 @@
minOccurs="0" maxOccurs="1" />
</xsd:all>
<xsd:attributeGroup ref="integration:channelAdapterAttributes"/>
<xsd:attribute name="kafka-producer-context-ref" use="required" type="xsd:string">
<xsd:attribute name="kafka-template" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[
Specifies the KafkaTemplate used to publish messages.
]]></xsd:documentation>
<xsd:appinfo>
<xsd:documentation>
Kafka producer context reference.
</xsd:documentation>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.integration.kafka.support.KafkaProducerContext"/>
<tool:expected-type type="org.springframework.kafka.core.KafkaTemplate" />
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
@@ -585,8 +121,7 @@
<xsd:element name="message-driven-channel-adapter">
<xsd:annotation>
<xsd:documentation>
The definition for the Spring Integration Kafka
KafkaMessageDrivenChannelAdapter.
Defines the Message Producing Endpoint for the KafkaMessageDrivenChannelAdapter.
</xsd:documentation>
</xsd:annotation>
<xsd:complexType>
@@ -612,170 +147,18 @@
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="key-decoder" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
A 'kafka.serializer.Decoder' bean reference for decoding the 'key'
of the received the 'kafka.message.Message'. Defaults to 'kafka.serializer.DefaultDecoder'
without 'kafka.utils.VerifiableProperties'.
</xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="kafka.serializer.Decoder"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="payload-decoder" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
A 'kafka.serializer.Decoder' bean reference for decoding the 'payload'
of the received the 'kafka.message.Message'. Defaults to 'kafka.serializer.DefaultDecoder'
without 'kafka.utils.VerifiableProperties'.
</xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="kafka.serializer.Decoder"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="listener-container" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
A 'org.springframework.integration.kafka.listener.KafkaMessageListenerContainer' bean reference.
Mutually exclusive with 'connection-factory', 'topics', 'offset-manager', 'error-handler',
'task-executor', 'concurrency', 'stop-timeout', 'max-fetch', 'queue-size'.
An 'org.springframework.kafka.listener.AbstractMessageListenerContainer' bean reference.
</xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.integration.kafka.listener.KafkaMessageListenerContainer"/>
<tool:expected-type type="org.springframework.kafka.listener.AbstractMessageListenerContainer"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="connection-factory" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
A 'org.springframework.integration.kafka.core.ConnectionFactory' bean reference.
Mutually exclusive with 'listener-container'.
</xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.integration.kafka.core.ConnectionFactory"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="topics" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
The comma-separated Kafka topics to listen to.
Mutually exclusive with 'listener-container'.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="offset-manager" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
A 'org.springframework.integration.kafka.listener.OffsetManager' bean reference.
Defaults to 'org.springframework.integration.kafka.listener.MetadataStoreOffsetManager'.
Mutually exclusive with 'listener-container'.
</xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.integration.kafka.listener.OffsetManager"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="error-handler" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
A 'org.springframework.integration.kafka.listener.ErrorHandler' bean reference.
Used from 'org.springframework.integration.kafka.listener.KafkaMessageListenerContainer'
to handle errors from 'kafka.message.Message' decoding and when the 'error-channel'
isn't provided to catch message producing errors.
Defaults to 'org.springframework.integration.kafka.listener.LoggingErrorHandler.
Mutually exclusive with 'listener-container'.
</xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.integration.kafka.listener.ErrorHandler"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="task-executor" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
A 'java.util.concurrent.Executor' bean reference to fetch messages from Kafka.
Defaults to the 'ThreadPoolExecutor' with the pool based on the number of 'partitionsByBroker'.
Mutually exclusive with 'listener-container'.
</xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="java.util.concurrent.Executor"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="concurrency" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
The number of internal concurrent
'org.springframework.integration.kafka.listener.QueueingMessageListenerInvoker'.
Defaults to '1'.
Mutually exclusive with 'listener-container'.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="stop-timeout" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
The maximum amount of time (in milliseconds) to wait for each 'org.springframework.integration.kafka.listener.QueueingMessageListenerInvoker' to finish before stopping.
Defaults to '1000'.
Mutually exclusive with 'listener-container'.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="max-fetch" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
The maximum amount of data (in bytes) that pollers will fetch in one round.
Defaults to '300 * 1024'.
Mutually exclusive with 'listener-container'.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="queue-size" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
The maximum number of messages that are buffered by each concurrent 'MessageListener' runner.
Increasing the value may increase throughput, but also increases the memory consumption.
Defaults to '1024'.
Mutually exclusive with 'listener-container'.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="auto-commit" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
When 'true', the adapter automatically commits the offset (also see `auto-commit-on-error`).
When 'false', the adapter inserts a 'kafka_acknowledgment` header allowing the user to manually
commit the offset using the `Acknowledgment.acknowledge()` method. Default 'true'.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="auto-commit-on-error" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
When 'true', the adapter automatically auto-commits only the offsets of successfully processed
messages if `auto-commit` is set to true. This means that only the offset of the last
successfully processed message is committed.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="use-context-message-builder" type="xsd:string">
<xsd:annotation>
<xsd:documentation>

54
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests-context.xml
View File

@@ -1,54 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
<int-kafka:zookeeper-connect id="zookeeperConnect" zk-connect="localhost:2181" zk-connection-timeout="6000"
zk-session-timeout="6000"
zk-sync-time="2000"/>
<bean id="valueDecoder" class="org.springframework.integration.kafka.serializer.avro.AvroReflectDatumBackedKafkaDecoder">
<constructor-arg type="java.lang.Class" value="java.lang.String"/>
</bean>
<bean id="consumerProperties" class="org.springframework.beans.factory.config.PropertiesFactoryBean">
<property name="properties">
<props>
<prop key="auto.offset.reset">largest</prop>
<prop key="socket.receive.buffer.bytes">10485760</prop> <!-- 10M -->
<prop key="fetch.message.max.bytes">5242880</prop>
<prop key="auto.commit.interval.ms">1000</prop>
</props>
</property>
</bean>
<util:properties id="placeholderProperties">
<prop key="topic.filter.pattern">foo</prop>
<prop key="topic.filter.streams">10</prop>
<prop key="topic.filter.exclude">true</prop>
</util:properties>
<context:property-placeholder properties-ref="placeholderProperties"/>
<int-kafka:consumer-context id="consumerContext"
consumer-timeout="4000"
zookeeper-connect="zookeeperConnect"
consumer-properties="consumerProperties">
<int-kafka:consumer-configurations>
<int-kafka:consumer-configuration group-id="default1"
value-decoder="valueDecoder"
key-decoder="valueDecoder"
max-messages="5000">
<int-kafka:topic-filter pattern="${topic.filter.pattern}"
streams="${topic.filter.streams}"
exclude="${topic.filter.exclude}"/>
</int-kafka:consumer-configuration>
</int-kafka:consumer-configurations>
</int-kafka:consumer-context>
</beans>

72
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests.java
View File

@@ -1,72 +0,0 @@
/*
* Copyright 2013-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import org.hamcrest.Matchers;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.integration.kafka.rule.KafkaRunning;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import kafka.consumer.Blacklist;
/**
* @author Soby Chacko
* @author Artem Bilan
* @author Gary Russell
* @since 0.5
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
@Deprecated
@SuppressWarnings("deprecation")
public class KafkaConsumerContextParserTests<K, V> {
@ClassRule
public static KafkaRunning kafkaRunning = KafkaRunning.isRunning();
@Autowired
private ApplicationContext appContext;
@Test
@SuppressWarnings("unchecked")
public void testConsumerContextConfiguration() {
final org.springframework.integration.kafka.support.KafkaConsumerContext<K, V> consumerContext =
appContext.getBean("consumerContext",
org.springframework.integration.kafka.support.KafkaConsumerContext.class);
Assert.assertNotNull(consumerContext);
org.springframework.integration.kafka.support.ConsumerConfiguration<K, V> cc
= consumerContext.getConsumerConfiguration("default1");
org.springframework.integration.kafka.support.ConsumerMetadata<K, V> cm = cc.getConsumerMetadata();
assertNotNull(cm);
org.springframework.integration.kafka.support.TopicFilterConfiguration topicFilterConfiguration =
cm.getTopicFilterConfiguration();
assertEquals("foo : 10", topicFilterConfiguration.toString());
assertThat(topicFilterConfiguration.getTopicFilter(), Matchers.instanceOf(Blacklist.class));
}
}

40
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests-context.xml
View File

@@ -1,40 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<import resource="kafkaInboundAdapterCommon-context.xml"/>
<int-kafka:zookeeper-connect id="zookeeperConnect" zk-connect="localhost:2181" zk-connection-timeout="6000"
zk-session-timeout="6000"
zk-sync-time="2000" />
<int-kafka:inbound-channel-adapter id="kafkaInboundChannelAdapter"
kafka-consumer-context-ref="consumerContext"
auto-startup="false"
channel="inputFromKafka">
<int:poller fixed-delay="10" time-unit="MILLISECONDS" max-messages-per-poll="5"/>
</int-kafka:inbound-channel-adapter>
<bean id="valueDecoder" class="org.springframework.integration.kafka.serializer.avro.AvroReflectDatumBackedKafkaDecoder">
<constructor-arg type="java.lang.Class" value="java.lang.String" />
</bean>
<int-kafka:consumer-context id="consumerContext"
consumer-timeout="4000"
zookeeper-connect="zookeeperConnect">
<int-kafka:consumer-configurations>
<int-kafka:consumer-configuration group-id="default"
value-decoder="valueDecoder"
key-decoder="valueDecoder"
max-messages="5000">
<int-kafka:topic id="test1" streams="4"/>
<int-kafka:topic id="test2" streams="4"/>
</int-kafka:consumer-configuration>
</int-kafka:consumer-configurations>
</int-kafka:consumer-context>
</beans>

58
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests.java
View File

@@ -1,58 +0,0 @@
/*
* Copyright 2013-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.integration.endpoint.SourcePollingChannelAdapter;
import org.springframework.integration.kafka.rule.KafkaRunning;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author Soby Chacko
* @author Gary Russell
* @since 0.5
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
@Deprecated
@SuppressWarnings("deprecation")
public class KafkaInboundAdapterParserTests {
@ClassRule
public static KafkaRunning kafkaRunning = KafkaRunning.isRunning();
@Autowired
private ApplicationContext appContext;
/**
* Test method for {@link org.springframework.integration.kafka.config.xml.KafkaInboundChannelAdapterParser#parseSource(org.w3c.dom.Element, org.springframework.beans.factory.xml.ParserContext)}.
*/
@Test
public void testParseSourceElementParserContext() throws Exception {
final SourcePollingChannelAdapter adapter = appContext.getBean("kafkaInboundChannelAdapter",
SourcePollingChannelAdapter.class);
Assert.assertNotNull(adapter);
Assert.assertFalse(adapter.isAutoStartup());
}
}

138
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParserTests-context.xml
View File

@@ -1,134 +1,38 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:task="http://www.springframework.org/schema/task"
xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:task="http://www.springframework.org/schema/task"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task.xsd">
http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
<context:property-placeholder/>
<bean id="connectionFactory" class="org.mockito.Mockito" factory-method="mock">
<constructor-arg value="org.springframework.integration.kafka.core.ConnectionFactory"/>
</bean>
<bean id="offsetManager" class="org.mockito.Mockito" factory-method="mock">
<constructor-arg value="org.springframework.integration.kafka.listener.OffsetManager"/>
</bean>
<bean id="errorHandler" class="org.mockito.Mockito" factory-method="mock">
<constructor-arg value="org.springframework.integration.kafka.listener.ErrorHandler"/>
</bean>
<bean id="keyDecoder" class="org.mockito.Mockito" factory-method="mock">
<constructor-arg value="kafka.serializer.Decoder"/>
</bean>
<bean id="payloadDecoder" class="org.mockito.Mockito" factory-method="mock">
<constructor-arg value="kafka.serializer.Decoder"/>
</bean>
<task:executor id="executor"/>
<int-kafka:message-driven-channel-adapter
id="kafkaListener"
listener-container="container1"
auto-startup="false"
phase="100"
send-timeout="5000"
channel="nullChannel"
error-channel="errorChannel"
error-handler="errorHandler"
connection-factory="connectionFactory"
key-decoder="keyDecoder"
payload-decoder="payloadDecoder"
offset-manager="offsetManager"
task-executor="executor"
stop-timeout="${stop.timeout:5000}"
queue-size="${queue.size:1024}"
concurrency="${concurrency:10}"
max-fetch="${max.fetch:1000}"
topics="${topics:foo,bar}" />
error-channel="errorChannel" />
<int-kafka:message-driven-channel-adapter
id="withMBFactoryOverrideAndId"
auto-startup="false"
phase="100"
send-timeout="5000"
channel="nullChannel"
error-channel="errorChannel"
error-handler="errorHandler"
connection-factory="connectionFactory"
key-decoder="keyDecoder"
payload-decoder="payloadDecoder"
offset-manager="offsetManager"
task-executor="executor"
queue-size="${queue.size:1024}"
concurrency="${concurrency:10}"
max-fetch="${max.fetch:1000}"
topics="${topics:foo,bar}"
auto-commit="false"
use-context-message-builder="true"
set-id="true"
set-timestamp="false" />
<int-kafka:message-driven-channel-adapter
id="withMBFactoryOverrideAndTS"
auto-startup="false"
phase="100"
send-timeout="5000"
channel="nullChannel"
error-channel="errorChannel"
error-handler="errorHandler"
connection-factory="connectionFactory"
key-decoder="keyDecoder"
payload-decoder="payloadDecoder"
offset-manager="offsetManager"
task-executor="executor"
queue-size="${queue.size:1024}"
concurrency="${concurrency:10}"
max-fetch="${max.fetch:1000}"
topics="${topics:foo,bar}"
use-context-message-builder="true"
set-id="false"
set-timestamp="true" />
<int-kafka:message-driven-channel-adapter
id="withOverrideIdTS"
auto-startup="false"
phase="100"
send-timeout="5000"
channel="nullChannel"
error-channel="errorChannel"
error-handler="errorHandler"
connection-factory="connectionFactory"
key-decoder="keyDecoder"
payload-decoder="payloadDecoder"
offset-manager="offsetManager"
task-executor="executor"
queue-size="${queue.size:1024}"
concurrency="${concurrency:10}"
max-fetch="${max.fetch:1000}"
topics="${topics:foo,bar}"
use-context-message-builder="false"
set-id="true"
set-timestamp="true"
auto-commit-on-error="true"/>
<!-- Invalid config
<bean id="container" class="org.mockito.Mockito" factory-method="mock">
<constructor-arg value="org.springframework.integration.kafka.listener.KafkaMessageListenerContainer"/>
<bean id="container1" class="org.springframework.kafka.listener.KafkaMessageListenerContainer">
<constructor-arg>
<bean class="org.springframework.kafka.core.DefaultKafkaConsumerFactory">
<constructor-arg>
<map>
<entry key="" value="" />
</map>
</constructor-arg>
</bean>
</constructor-arg>
<constructor-arg name="topics" value="foo" />
</bean>
<int-kafka:message-driven-channel-adapter
channel="nullChannel"
topics="foo"
listener-container="container"/>
-->
</beans>

151
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParserTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2015 the original author or authors
* Copyright 2015-2016 the original author or authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,47 +16,22 @@
package org.springframework.integration.kafka.config.xml;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.lang.reflect.Method;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicReference;
import org.hamcrest.Matchers;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.integration.channel.NullChannel;
import org.springframework.integration.channel.PublishSubscribeChannel;
import org.springframework.integration.kafka.core.ConnectionFactory;
import org.springframework.integration.kafka.core.KafkaMessageMetadata;
import org.springframework.integration.kafka.core.Partition;
import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter;
import org.springframework.integration.kafka.listener.Acknowledgment;
import org.springframework.integration.kafka.listener.ErrorHandler;
import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer;
import org.springframework.integration.kafka.listener.OffsetManager;
import org.springframework.integration.kafka.support.KafkaHeaders;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.messaging.Message;
import org.springframework.messaging.support.GenericMessage;
import org.springframework.kafka.listener.KafkaMessageListenerContainer;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.util.ReflectionUtils;
import org.springframework.util.ReflectionUtils.MethodCallback;
import org.springframework.util.ReflectionUtils.MethodFilter;
import kafka.serializer.Decoder;
/**
* @author Artem Bilan.
@@ -66,24 +41,6 @@ import kafka.serializer.Decoder;
@ContextConfiguration
public class KafkaMessageDrivenChannelAdapterParserTests {
@Autowired
private ConnectionFactory connectionFactory;
@Autowired
private OffsetManager offsetManager;
@Autowired
private ErrorHandler errorHandler;
@Autowired
private Executor executor;
@Autowired
private Decoder<?> keyDecoder;
@Autowired
private Decoder<?> payloadDecoder;
@Autowired
private NullChannel nullChannel;
@@ -91,16 +48,7 @@ public class KafkaMessageDrivenChannelAdapterParserTests {
private PublishSubscribeChannel errorChannel;
@Autowired
private KafkaMessageDrivenChannelAdapter kafkaListener;
@Autowired
private KafkaMessageDrivenChannelAdapter withMBFactoryOverrideAndId;
@Autowired
private KafkaMessageDrivenChannelAdapter withMBFactoryOverrideAndTS;
@Autowired
private KafkaMessageDrivenChannelAdapter withOverrideIdTS;
private KafkaMessageDrivenChannelAdapter<?, ?> kafkaListener;
@Test
public void testKafkaMessageDrivenChannelAdapterParser() throws Exception {
@@ -109,99 +57,10 @@ public class KafkaMessageDrivenChannelAdapterParserTests {
assertEquals(100, this.kafkaListener.getPhase());
assertSame(this.nullChannel, TestUtils.getPropertyValue(this.kafkaListener, "outputChannel"));
assertSame(this.errorChannel, TestUtils.getPropertyValue(this.kafkaListener, "errorChannel"));
assertSame(this.keyDecoder, TestUtils.getPropertyValue(this.kafkaListener, "keyDecoder"));
assertSame(this.payloadDecoder, TestUtils.getPropertyValue(this.kafkaListener, "payloadDecoder"));
KafkaMessageListenerContainer container =
KafkaMessageListenerContainer<?, ?> container =
TestUtils.getPropertyValue(this.kafkaListener, "messageListenerContainer",
KafkaMessageListenerContainer.class);
assertSame(this.connectionFactory, TestUtils.getPropertyValue(container, "kafkaTemplate.connectionFactory"));
assertSame(this.offsetManager, container.getOffsetManager());
assertSame(this.errorHandler, container.getErrorHandler());
assertSame(this.executor, container.getFetchTaskExecutor());
assertEquals(10, container.getConcurrency());
assertEquals(1000, container.getMaxFetch());
assertEquals(1024, container.getQueueSize());
assertEquals(5000, container.getStopTimeout());
assertArrayEquals(new String[] {"foo", "bar"}, TestUtils.getPropertyValue(container, "topics", String[].class));
assertOverrides(this.kafkaListener, false, false, false, true, false);
assertOverrides(this.withMBFactoryOverrideAndId, true, true, false, false, false);
assertOverrides(this.withMBFactoryOverrideAndTS, true, false, true, true, false);
assertOverrides(this.withOverrideIdTS, false, true, true, true, true);
final AtomicReference<Method> toMessage = new AtomicReference<Method>();
ReflectionUtils.doWithMethods(KafkaMessageDrivenChannelAdapter.class, new MethodCallback() {
@Override
public void doWith(Method method) throws IllegalArgumentException, IllegalAccessException {
method.setAccessible(true);
toMessage.set(method);
}
},
new MethodFilter() {
@Override
public boolean matches(Method method) {
return method.getName().equals("toMessage");
}
});
Message<?> m = getAMessageFrom(this.kafkaListener, toMessage.get());
assertNull(m.getHeaders().getId());
assertNull(m.getHeaders().getTimestamp());
assertNull(m.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT));
assertRest(m);
m = getAMessageFrom(this.withMBFactoryOverrideAndId, toMessage.get());
assertThat(m, Matchers.instanceOf(GenericMessage.class));
assertNotNull(m.getHeaders().getId());
assertNotNull(m.getHeaders().getTimestamp());
assertNotNull(m.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT));
assertRest(m);
m = getAMessageFrom(this.withMBFactoryOverrideAndTS, toMessage.get());
assertThat(m, Matchers.instanceOf(GenericMessage.class));
assertNotNull(m.getHeaders().getId());
assertNotNull(m.getHeaders().getTimestamp());
assertNull(m.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT));
assertRest(m);
m = getAMessageFrom(this.withOverrideIdTS, toMessage.get());
assertNotNull(m.getHeaders().getId());
assertNotNull(m.getHeaders().getTimestamp());
assertNull(m.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT));
assertRest(m);
}
private void assertOverrides(KafkaMessageDrivenChannelAdapter kafkaListener, boolean mbf, boolean id, boolean ts,
boolean ac, boolean ace) {
assertThat(TestUtils.getPropertyValue(kafkaListener, "autoCommitOffset", Boolean.class), equalTo(ac));
assertThat(TestUtils.getPropertyValue(kafkaListener, "useMessageBuilderFactory", Boolean.class), equalTo(mbf));
assertThat(TestUtils.getPropertyValue(kafkaListener, "generateMessageId", Boolean.class), equalTo(id));
assertThat(TestUtils.getPropertyValue(kafkaListener, "generateTimestamp", Boolean.class), equalTo(ts));
Object messageListenerContainer = TestUtils.getPropertyValue(kafkaListener, "messageListenerContainer");
assertEquals(ace, TestUtils.getPropertyValue(messageListenerContainer,"autoCommitOnError", Boolean.class));
}
private void assertRest(Message<?> m) {
assertEquals("bar", m.getPayload());
assertEquals("foo", m.getHeaders().get(KafkaHeaders.MESSAGE_KEY));
assertEquals("topic", m.getHeaders().get(KafkaHeaders.TOPIC));
assertEquals(42, m.getHeaders().get(KafkaHeaders.PARTITION_ID));
assertEquals(1L, m.getHeaders().get(KafkaHeaders.OFFSET));
assertEquals(2L, m.getHeaders().get(KafkaHeaders.NEXT_OFFSET));
}
private Message<?> getAMessageFrom(KafkaMessageDrivenChannelAdapter adapter, Method toMessage) throws Exception {
KafkaMessageMetadata meta = mock(KafkaMessageMetadata.class);
Partition partition = mock(Partition.class);
when(partition.getTopic()).thenReturn("topic");
when(partition.getId()).thenReturn(42);
when(meta.getPartition()).thenReturn(partition);
when(meta.getOffset()).thenReturn(1L);
when(meta.getNextOffset()).thenReturn(2L);
Acknowledgment ack = mock(Acknowledgment.class);
return (Message<?>) toMessage.invoke(adapter, "foo", "bar", meta, ack);
assertNotNull(container);
}
}

69
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMultiConsumerContextParserTests-context.xml
View File

@@ -1,69 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka
http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<context:property-placeholder/>
<int-kafka:zookeeper-connect id="zookeeperConnect" zk-connect="localhost:2181" zk-connection-timeout="6000"
zk-session-timeout="6000"
zk-sync-time="2000"/>
<bean id="valueDecoder"
class="org.springframework.integration.kafka.serializer.avro.AvroReflectDatumBackedKafkaDecoder">
<constructor-arg type="java.lang.Class" value="java.lang.String"/>
</bean>
<bean id="consumerProperties" class="org.springframework.beans.factory.config.PropertiesFactoryBean">
<property name="properties">
<props>
<prop key="auto.offset.reset">largest</prop>
<prop key="socket.receive.buffer.bytes">10485760</prop>
<!-- 10M -->
<prop key="fetch.message.max.bytes">5242880</prop>
<prop key="auto.commit.interval.ms">1000</prop>
</props>
</property>
</bean>
<int-kafka:consumer-context id="consumerContext1"
consumer-timeout="4000"
zookeeper-connect="zookeeperConnect"
consumer-properties="consumerProperties">
<int-kafka:consumer-configurations>
<int-kafka:consumer-configuration group-id="${groupId:default1}"
value-decoder="valueDecoder"
key-decoder="valueDecoder"
max-messages="5000">
<int-kafka:topic id="test1" streams="${streams:3}"/>
<int-kafka:topic id="test2" streams="4"/>
</int-kafka:consumer-configuration>
<int-kafka:consumer-configuration group-id="${groupId:default2}"
value-decoder="valueDecoder"
key-decoder="valueDecoder"
max-messages="5000">
<int-kafka:topic id="test3" streams="${streams:1}"/>
</int-kafka:consumer-configuration>
</int-kafka:consumer-configurations>
</int-kafka:consumer-context>
<int-kafka:consumer-context id="consumerContext2"
consumer-timeout="4000"
zookeeper-connect="zookeeperConnect"
consumer-properties="consumerProperties">
<int-kafka:consumer-configurations>
<int-kafka:consumer-configuration group-id="${groupId:default1}"
value-decoder="valueDecoder"
key-decoder="valueDecoder"
max-messages="5000">
<int-kafka:topic id="test4" streams="${streams:3}"/>
<int-kafka:topic id="test5" streams="4"/>
</int-kafka:consumer-configuration>
</int-kafka:consumer-configurations>
</int-kafka:consumer-context>
</beans>

86
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMultiConsumerContextParserTests.java
View File

@@ -1,86 +0,0 @@
/*
* Copyright 2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.integration.kafka.rule.KafkaRunning;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author Ilayaperumal Gopinathan
* @author Gary Russell
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
@Deprecated
@SuppressWarnings("deprecation")
public class KafkaMultiConsumerContextParserTests<K,V> {
@ClassRule
public static KafkaRunning kafkaRunning = KafkaRunning.isRunning();
@Autowired
private ApplicationContext appContext;
@SuppressWarnings("unchecked")
@Test
public void testMultiConsumerContexts() {
final org.springframework.integration.kafka.support.KafkaConsumerContext<K,V> consumerContext1 =
appContext.getBean("consumerContext1",
org.springframework.integration.kafka.support.KafkaConsumerContext.class);
Assert.assertNotNull(consumerContext1);
final org.springframework.integration.kafka.support.KafkaConsumerContext<K,V> consumerContext2 =
appContext.getBean("consumerContext2",
org.springframework.integration.kafka.support.KafkaConsumerContext.class);
Assert.assertNotNull(consumerContext2);
}
@SuppressWarnings("unchecked")
@Test
public void testConsumerContextConfigurations() {
final org.springframework.integration.kafka.support.KafkaConsumerContext<K,V> consumerContext =
appContext.getBean("consumerContext1",
org.springframework.integration.kafka.support.KafkaConsumerContext.class);
Assert.assertNotNull(consumerContext);
final org.springframework.integration.kafka.support.ConsumerConfiguration<K,V> cc =
consumerContext.getConsumerConfiguration("default1");
final org.springframework.integration.kafka.support.ConsumerMetadata<K,V> cm = cc.getConsumerMetadata();
Assert.assertTrue(cm.getTopicStreamMap().get("test1") == 3);
Assert.assertTrue(cm.getTopicStreamMap().get("test2") == 4);
Assert.assertNotNull(cm);
final org.springframework.integration.kafka.support.ConsumerConfiguration<K,V> cc2 =
consumerContext.getConsumerConfiguration("default2");
final org.springframework.integration.kafka.support.ConsumerMetadata<K,V> cm2 =
cc2.getConsumerMetadata();
Assert.assertTrue(cm2.getTopicStreamMap().get("test3") == 1);
Assert.assertNotNull(cm2);
final org.springframework.integration.kafka.support.KafkaConsumerContext<K,V> consumerContext2 =
appContext.getBean("consumerContext2",
org.springframework.integration.kafka.support.KafkaConsumerContext.class);
Assert.assertNotNull(consumerContext2);
final org.springframework.integration.kafka.support.ConsumerConfiguration<K,V> otherCC =
consumerContext2.getConsumerConfiguration("default1");
final org.springframework.integration.kafka.support.ConsumerMetadata<K,V> otherCM =
otherCC.getConsumerMetadata();
Assert.assertTrue(otherCM.getTopicStreamMap().get("test4") == 3);
}
}

51
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests-context.xml
View File

@@ -9,58 +9,31 @@
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task.xsd">
<int:channel id="inputToKafka">
<int:queue/>
</int:channel>
<int:channel id="inputToKafka" />
<int-kafka:outbound-channel-adapter id="kafkaOutboundChannelAdapter"
kafka-producer-context-ref="kafkaProducerContext"
kafka-template="template"
auto-startup="false"
channel="inputToKafka"
order="3"
topic="foo"
message-key-expression="'bar'"
partition-id-expression="2">
<int:poller fixed-delay="1000" time-unit="MILLISECONDS" receive-timeout="0" task-executor="taskExecutor"/>
<int-kafka:request-handler-advice-chain>
<bean class="org.springframework.integration.handler.advice.RequestHandlerCircuitBreakerAdvice" />
</int-kafka:request-handler-advice-chain>
</int-kafka:outbound-channel-adapter>
<int-kafka:outbound-channel-adapter id="kafkaOutboundChannelAdapterWithHeaderRoutingDisabled"
kafka-producer-context-ref="kafkaProducerContext"
auto-startup="false"
channel="inputToKafka"
order="3"
topic="foo"
message-key-expression="'bar'"
partition-id-expression="2"
enable-header-routing="false">
<int:poller fixed-delay="1000" time-unit="MILLISECONDS" receive-timeout="0" task-executor="taskExecutor"/>
</int-kafka:outbound-channel-adapter>
<task:executor id="taskExecutor" pool-size="5" keep-alive="120" queue-capacity="500"/>
<bean id="kafkaEncoder"
class="org.springframework.integration.kafka.serializer.avro.AvroReflectDatumBackedKafkaEncoder">
<constructor-arg value="java.lang.String"/>
<bean id="template" class="org.springframework.kafka.core.KafkaTemplate">
<constructor-arg>
<bean class="org.springframework.kafka.core.DefaultKafkaProducerFactory">
<constructor-arg>
<map>
<entry key="" value="" />
</map>
</constructor-arg>
</bean>
</constructor-arg>
</bean>
<int-kafka:producer-context id="kafkaProducerContext">
<int-kafka:producer-configurations>
<int-kafka:producer-configuration broker-list="localhost:9092"
key-class-type="java.lang.String"
value-class-type="java.lang.String"
topic="test1"
value-encoder="kafkaEncoder"
key-encoder="kafkaEncoder"
compression-type="none"/>
<int-kafka:producer-configuration broker-list="localhost:9092"
topic="test2"
value-encoder="kafkaEncoder"
key-encoder="kafkaEncoder"
compression-type="none"/>
</int-kafka:producer-configurations>
</int-kafka:producer-context>
</beans>

36
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests.java
View File

@@ -16,29 +16,16 @@
package org.springframework.integration.kafka.config.xml;
import static org.hamcrest.Matchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.aop.Advisor;
import org.springframework.aop.framework.Advised;
import org.springframework.aop.support.AopUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.integration.endpoint.PollingConsumer;
import org.springframework.integration.handler.advice.RequestHandlerCircuitBreakerAdvice;
import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler;
import org.springframework.integration.kafka.rule.KafkaEmbedded;
import org.springframework.integration.kafka.rule.KafkaRule;
import org.springframework.integration.kafka.support.KafkaProducerContext;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.messaging.MessageHandler;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@@ -54,38 +41,19 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@DirtiesContext
public class KafkaOutboundAdapterParserTests {
@ClassRule
public static KafkaRule kafkaRunning = new KafkaEmbedded(1);
@Autowired
private ApplicationContext appContext;
@Test
public void testOutboundAdapterConfiguration() {
PollingConsumer pollingConsumer = this.appContext.getBean("kafkaOutboundChannelAdapter", PollingConsumer.class);
KafkaProducerMessageHandler messageHandler
= this.appContext.getBean("org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler#0",
KafkaProducerMessageHandler.class);
assertNotNull(pollingConsumer);
KafkaProducerMessageHandler<?, ?> messageHandler
= this.appContext.getBean("kafkaOutboundChannelAdapter.handler", KafkaProducerMessageHandler.class);
assertNotNull(messageHandler);
assertEquals(messageHandler.getOrder(), 3);
assertEquals("foo", TestUtils.getPropertyValue(messageHandler, "topicExpression.literalValue"));
assertEquals("'bar'", TestUtils.getPropertyValue(messageHandler, "messageKeyExpression.expression"));
assertEquals("2", TestUtils.getPropertyValue(messageHandler, "partitionIdExpression.expression"));
assertEquals(true, TestUtils.getPropertyValue(messageHandler, "enableHeaderRouting"));
KafkaProducerContext producerContext = messageHandler.getKafkaProducerContext();
assertNotNull(producerContext);
assertEquals(producerContext.getProducerConfigurations().size(), 2);
KafkaProducerMessageHandler messageHandler2
= this.appContext.getBean("org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler#1",
KafkaProducerMessageHandler.class);
assertEquals(false, TestUtils.getPropertyValue(messageHandler2, "enableHeaderRouting"));
MessageHandler advisedHandler = TestUtils.getPropertyValue(pollingConsumer, "handler", MessageHandler.class);
assertTrue(AopUtils.isJdkDynamicProxy(advisedHandler));
Advisor[] advisors = ((Advised) advisedHandler).getAdvisors();
assertEquals(1, advisors.length);
assertThat(advisors[0].getAdvice(), instanceOf(RequestHandlerCircuitBreakerAdvice.class));
}
}

64
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests-context.xml
View File

@@ -1,64 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
<bean id="producerProperties" class="org.springframework.beans.factory.config.PropertiesFactoryBean">
<property name="properties">
<props>
<prop key="metadata.max.age.ms">3600000</prop>
<prop key="retries">5</prop>
<prop key="send.buffer.bytes">5242880</prop>
</props>
</property>
</bean>
<util:properties id="placeholderProperties">
<prop key="brokerList1">localhost:9092</prop>
<prop key="brokerList2">localhost:9091</prop>
<prop key="topic1">test1</prop>
<prop key="topic2">test2</prop>
</util:properties>
<context:property-placeholder properties-ref="placeholderProperties"/>
<int-kafka:producer-context id="producerContext" producer-properties="producerProperties"
auto-startup="false" phase="123">
<int-kafka:producer-configurations>
<int-kafka:producer-configuration broker-list="${brokerList1}"
key-class-type="java.lang.String"
value-class-type="java.lang.String"
key-encoder="valueEncoder"
value-encoder="valueEncoder"
topic="${topic1}"
compression-type="none"/>
<int-kafka:producer-configuration broker-list="${brokerList2}"
topic="${topic2}"
key-class-type="java.lang.String"
value-class-type="java.lang.String"
key-serializer="stringSerializer"
value-serializer="stringSerializer"
batch-bytes="9876"
conversion-service="conversionService"
producer-listener="producerListener"
charset="cp1251"
compression-type="none"/>
</int-kafka:producer-configurations>
</int-kafka:producer-context>
<bean id="valueEncoder" class="org.springframework.integration.kafka.serializer.avro.AvroReflectDatumBackedKafkaEncoder">
<constructor-arg value="java.lang.String" />
</bean>
<bean id="producerListener" class="org.springframework.integration.kafka.support.LoggingProducerListener"/>
<bean id="stringSerializer" class="org.apache.kafka.common.serialization.StringSerializer"/>
<bean id="conversionService" class="org.springframework.integration.kafka.config.xml.KafkaProducerContextParserTests.StubConversionService"/>
</beans>

143
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests.java
View File

@@ -1,143 +0,0 @@
/*
* Copyright 2013-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertSame;
import java.util.Map;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.common.serialization.Serializer;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.DirectFieldAccessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.core.convert.ConversionService;
import org.springframework.core.convert.TypeDescriptor;
import org.springframework.integration.kafka.rule.KafkaEmbedded;
import org.springframework.integration.kafka.rule.KafkaRule;
import org.springframework.integration.kafka.support.KafkaProducerContext;
import org.springframework.integration.kafka.support.ProducerConfiguration;
import org.springframework.integration.kafka.support.ProducerListener;
import org.springframework.integration.kafka.support.ProducerMetadata;
import org.springframework.integration.kafka.util.EncoderAdaptingSerializer;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import kafka.serializer.Encoder;
/**
* @author Soby Chacko
* @author Gary Russell
* @since 0.5
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
public class KafkaProducerContextParserTests {
@ClassRule
public static KafkaRule kafkaRule = new KafkaEmbedded(1);
@Autowired
private ApplicationContext appContext;
@Test
@SuppressWarnings({"unchecked","rawtypes"})
public void testProducerContextConfiguration(){
final KafkaProducerContext producerContext = appContext.getBean("producerContext", KafkaProducerContext.class);
Assert.assertNotNull(producerContext);
final Map<String, ProducerConfiguration<?,?>> producerConfigurations = producerContext.getProducerConfigurations();
assertEquals(producerConfigurations.size(), 2);
final ProducerConfiguration<?,?> producerConfigurationTest1 = producerConfigurations.get("test1");
Assert.assertNotNull(producerConfigurationTest1);
final ProducerMetadata<?,?> producerMetadataTest1 = producerConfigurationTest1.getProducerMetadata();
assertEquals(producerMetadataTest1.getTopic(), "test1");
assertEquals(producerMetadataTest1.getCompressionType(), ProducerMetadata.CompressionType.none);
assertEquals(producerMetadataTest1.getKeyClassType(), java.lang.String.class);
assertEquals(producerMetadataTest1.getValueClassType(), java.lang.String.class);
final Encoder<?> valueEncoder = appContext.getBean("valueEncoder", Encoder.class);
Assert.assertThat((Class) producerMetadataTest1.getKeySerializer().getClass(), equalTo((Class) EncoderAdaptingSerializer.class));
Assert.assertThat(((EncoderAdaptingSerializer) producerMetadataTest1.getKeySerializer()).getEncoder(), equalTo((Encoder) valueEncoder));
Assert.assertThat((Class)producerMetadataTest1.getValueSerializer().getClass(), equalTo((Class)EncoderAdaptingSerializer.class));
Assert.assertThat(((EncoderAdaptingSerializer)producerMetadataTest1.getValueSerializer()).getEncoder(), equalTo((Encoder)valueEncoder));
DirectFieldAccessor directFieldAccessor = new DirectFieldAccessor(producerConfigurationTest1);
Producer<?,?> producerTest1 = (Producer<?, ?>) directFieldAccessor.getPropertyValue("producer");
assertEquals(producerConfigurationTest1.getProducerMetadata(), producerMetadataTest1);
final ProducerConfiguration<?,?> producerConfigurationTest2 = producerConfigurations.get("test2");
Assert.assertNotNull(producerConfigurationTest2);
final ProducerMetadata<?,?> producerMetadataTest2 = producerConfigurationTest2.getProducerMetadata();
assertEquals(producerMetadataTest2.getTopic(), "test2");
assertEquals(producerMetadataTest2.getCompressionType(), ProducerMetadata.CompressionType.none);
DirectFieldAccessor directFieldAccessor2 = new DirectFieldAccessor(producerConfigurationTest2);
Producer<?,?> producerTest2 = (Producer<?, ?>) directFieldAccessor2.getPropertyValue("producer");
assertEquals(producerConfigurationTest2.getProducerMetadata(), producerMetadataTest2);
final Serializer<?> stringSerializer = appContext.getBean("stringSerializer", Serializer.class);
assertSame(stringSerializer, producerConfigurationTest2.getProducerMetadata().getKeySerializer());
assertSame(stringSerializer, producerConfigurationTest2.getProducerMetadata().getValueSerializer());
final ConversionService conversionService = appContext.getBean("conversionService", ConversionService.class);
ConversionService configuredConversionService = (ConversionService) directFieldAccessor2.getPropertyValue("conversionService");
assertSame(conversionService, configuredConversionService);
final ProducerListener producerListener = appContext.getBean("producerListener", ProducerListener.class);
ProducerListener configuredProducerListener = (ProducerListener) directFieldAccessor2.getPropertyValue("producerListener");
assertSame(producerListener, configuredProducerListener);
assertEquals(9876,producerConfigurationTest2.getProducerMetadata().getBatchBytes());
assertFalse(TestUtils.getPropertyValue(producerContext, "autoStartup", Boolean.class));
assertEquals(123, TestUtils.getPropertyValue(producerContext, "phase"));
assertEquals("windows-1251", producerConfigurationTest2.getProducerMetadata().getCharset().name());
}
public static class StubConversionService implements ConversionService {
@Override
public boolean canConvert(Class<?> sourceType, Class<?> targetType) {
return false;
}
@Override
public boolean canConvert(TypeDescriptor sourceType, TypeDescriptor targetType) {
return false;
}
@Override
public <T> T convert(Object source, Class<T> targetType) {
return null;
}
@Override
public Object convert(Object source, TypeDescriptor sourceType, TypeDescriptor targetType) {
return null;
}
}
}

14
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests-context.xml
View File

@@ -1,14 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xsi:schemaLocation="
http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<int-kafka:zookeeper-connect id="zookeeperConnect" zk-connect="localhost:2181" zk-connection-timeout="10000"
zk-session-timeout="10000"
zk-sync-time="200" />
<int-kafka:zookeeper-connect id="defaultZookeeperConnect" />
</beans>

65
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests.java
View File

@@ -1,65 +0,0 @@
/*
* Copyright 2013-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.integration.kafka.core.ZookeeperConnectDefaults;
import org.springframework.integration.kafka.rule.KafkaRunning;
import org.springframework.integration.kafka.support.ZookeeperConnect;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author Soby Chacko
* @author Gary Russell
* @since 0.5
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
public class ZookeeperConnectParserTests {
@ClassRule
public static KafkaRunning kafkaRunning = KafkaRunning.isRunning();
@Autowired
private ApplicationContext appContext;
@Test
public void testCustomKafkaBrokerConfiguration() {
final ZookeeperConnect broker = appContext.getBean("zookeeperConnect", ZookeeperConnect.class);
Assert.assertEquals("localhost:2181", broker.getZkConnect());
Assert.assertEquals("10000", broker.getZkConnectionTimeout());
Assert.assertEquals("10000", broker.getZkSessionTimeout());
Assert.assertEquals("200", broker.getZkSyncTime());
}
@Test
public void testDefaultKafkaBrokerConfiguration() {
final ZookeeperConnect broker = appContext.getBean("defaultZookeeperConnect", ZookeeperConnect.class);
Assert.assertEquals(ZookeeperConnectDefaults.ZK_CONNECT, broker.getZkConnect());
Assert.assertEquals(ZookeeperConnectDefaults.ZK_CONNECTION_TIMEOUT, broker.getZkConnectionTimeout());
Assert.assertEquals(ZookeeperConnectDefaults.ZK_SESSION_TIMEOUT, broker.getZkSessionTimeout());
Assert.assertEquals(ZookeeperConnectDefaults.ZK_SYNC_TIME, broker.getZkSyncTime());
}
}

Some files were not shown because too many files have changed in this diff Show More