Kafka - Update to S-I-K 2.0
Polishing
This commit is contained in:
Gary Russell
committed by
Artem Bilan
Artem Bilan
parent
689279656e
commit
f943f5e20f
@@ -16,10 +16,15 @@
|
||||
|
||||
package org.springframework.integration.samples.kafka;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.I0Itec.zkclient.ZkClient;
|
||||
import org.apache.kafka.clients.consumer.ConsumerConfig;
|
||||
import org.apache.kafka.clients.producer.ProducerConfig;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.kafka.common.serialization.StringDeserializer;
|
||||
import org.apache.kafka.common.serialization.StringSerializer;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
@@ -31,29 +36,25 @@ import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.expression.common.LiteralExpression;
|
||||
import org.springframework.integration.annotation.ServiceActivator;
|
||||
import org.springframework.integration.channel.QueueChannel;
|
||||
import org.springframework.integration.kafka.core.BrokerAddress;
|
||||
import org.springframework.integration.kafka.core.BrokerAddressListConfiguration;
|
||||
import org.springframework.integration.kafka.core.ConnectionFactory;
|
||||
import org.springframework.integration.kafka.core.DefaultConnectionFactory;
|
||||
import org.springframework.integration.kafka.core.Partition;
|
||||
import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter;
|
||||
import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer;
|
||||
import org.springframework.integration.kafka.listener.KafkaTopicOffsetManager;
|
||||
import org.springframework.integration.kafka.listener.OffsetManager;
|
||||
import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler;
|
||||
import org.springframework.integration.kafka.serializer.common.StringDecoder;
|
||||
import org.springframework.integration.kafka.support.KafkaProducerContext;
|
||||
import org.springframework.integration.kafka.support.ProducerConfiguration;
|
||||
import org.springframework.integration.kafka.support.ProducerFactoryBean;
|
||||
import org.springframework.integration.kafka.support.ProducerMetadata;
|
||||
import org.springframework.integration.kafka.support.ZookeeperConnect;
|
||||
import org.springframework.integration.kafka.util.TopicUtils;
|
||||
import org.springframework.kafka.core.ConsumerFactory;
|
||||
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
|
||||
import org.springframework.kafka.core.DefaultKafkaProducerFactory;
|
||||
import org.springframework.kafka.core.KafkaTemplate;
|
||||
import org.springframework.kafka.core.ProducerFactory;
|
||||
import org.springframework.kafka.listener.KafkaMessageListenerContainer;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.MessageChannel;
|
||||
import org.springframework.messaging.MessageHandler;
|
||||
import org.springframework.messaging.PollableChannel;
|
||||
import org.springframework.messaging.support.GenericMessage;
|
||||
|
||||
import kafka.admin.AdminUtils;
|
||||
import kafka.common.TopicExistsException;
|
||||
import kafka.utils.ZKStringSerializer$;
|
||||
import kafka.utils.ZkUtils;
|
||||
|
||||
/**
|
||||
* @author Gary Russell
|
||||
* @since 4.2
|
||||
@@ -95,64 +96,54 @@ public class Application {
|
||||
@ServiceActivator(inputChannel = "toKafka")
|
||||
@Bean
|
||||
public MessageHandler handler() throws Exception {
|
||||
KafkaProducerMessageHandler handler = new KafkaProducerMessageHandler(producerContext());
|
||||
KafkaProducerMessageHandler<String, String> handler =
|
||||
new KafkaProducerMessageHandler<>(kafkaTemplate());
|
||||
handler.setTopicExpression(new LiteralExpression(this.topic));
|
||||
handler.setMessageKeyExpression(new LiteralExpression(this.messageKey));
|
||||
return handler;
|
||||
}
|
||||
|
||||
@Bean
|
||||
public ConnectionFactory kafkaBrokerConnectionFactory() throws Exception {
|
||||
return new DefaultConnectionFactory(kafkaConfiguration());
|
||||
public KafkaTemplate<String, String> kafkaTemplate() {
|
||||
return new KafkaTemplate<>(producerFactory());
|
||||
}
|
||||
|
||||
@Bean
|
||||
public org.springframework.integration.kafka.core.Configuration kafkaConfiguration() {
|
||||
BrokerAddressListConfiguration configuration = new BrokerAddressListConfiguration(
|
||||
BrokerAddress.fromAddress(this.brokerAddress));
|
||||
configuration.setSocketTimeout(500);
|
||||
return configuration;
|
||||
public ProducerFactory<String, String> producerFactory() {
|
||||
Map<String, Object> props = new HashMap<>();
|
||||
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, this.brokerAddress);
|
||||
props.put(ProducerConfig.RETRIES_CONFIG, 0);
|
||||
props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
|
||||
props.put(ProducerConfig.LINGER_MS_CONFIG, 1);
|
||||
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432);
|
||||
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
|
||||
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
|
||||
return new DefaultKafkaProducerFactory<>(props);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public KafkaProducerContext producerContext() throws Exception {
|
||||
KafkaProducerContext kafkaProducerContext = new KafkaProducerContext();
|
||||
ProducerMetadata<String, String> producerMetadata = new ProducerMetadata<>(this.topic, String.class,
|
||||
String.class, new StringSerializer(), new StringSerializer());
|
||||
Properties props = new Properties();
|
||||
props.put("linger.ms", "1000");
|
||||
ProducerFactoryBean<String, String> producer =
|
||||
new ProducerFactoryBean<>(producerMetadata, this.brokerAddress, props);
|
||||
ProducerConfiguration<String, String> config =
|
||||
new ProducerConfiguration<>(producerMetadata, producer.getObject());
|
||||
Map<String, ProducerConfiguration<?, ?>> producerConfigurationMap =
|
||||
Collections.<String, ProducerConfiguration<?, ?>>singletonMap(this.topic, config);
|
||||
kafkaProducerContext.setProducerConfigurations(producerConfigurationMap);
|
||||
return kafkaProducerContext;
|
||||
public KafkaMessageListenerContainer<String, String> container() throws Exception {
|
||||
return new KafkaMessageListenerContainer<>(consumerFactory(), new TopicPartition(this.topic, 0));
|
||||
}
|
||||
|
||||
@Bean
|
||||
public OffsetManager offsetManager() {
|
||||
return new KafkaTopicOffsetManager(new ZookeeperConnect(this.zookeeperConnect), "si-offsets");
|
||||
public ConsumerFactory<String, String> consumerFactory() {
|
||||
Map<String, Object> props = new HashMap<>();
|
||||
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, this.brokerAddress);
|
||||
props.put(ConsumerConfig.GROUP_ID_CONFIG, "siTestGroup");
|
||||
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true);
|
||||
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 100);
|
||||
props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 15000);
|
||||
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
|
||||
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
|
||||
return new DefaultKafkaConsumerFactory<>(props);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public KafkaMessageListenerContainer container(OffsetManager offsetManager) throws Exception {
|
||||
final KafkaMessageListenerContainer kafkaMessageListenerContainer = new KafkaMessageListenerContainer(
|
||||
kafkaBrokerConnectionFactory(), new Partition(this.topic, 0));
|
||||
kafkaMessageListenerContainer.setOffsetManager(offsetManager);
|
||||
kafkaMessageListenerContainer.setMaxFetch(100);
|
||||
kafkaMessageListenerContainer.setConcurrency(1);
|
||||
return kafkaMessageListenerContainer;
|
||||
}
|
||||
|
||||
@Bean
|
||||
public KafkaMessageDrivenChannelAdapter adapter(KafkaMessageListenerContainer container) {
|
||||
KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter =
|
||||
new KafkaMessageDrivenChannelAdapter(container);
|
||||
StringDecoder decoder = new StringDecoder();
|
||||
kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder);
|
||||
kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder);
|
||||
public KafkaMessageDrivenChannelAdapter<String, String>
|
||||
adapter(KafkaMessageListenerContainer<String, String> container) {
|
||||
KafkaMessageDrivenChannelAdapter<String, String> kafkaMessageDrivenChannelAdapter =
|
||||
new KafkaMessageDrivenChannelAdapter<>(container);
|
||||
kafkaMessageDrivenChannelAdapter.setOutputChannel(received());
|
||||
return kafkaMessageDrivenChannelAdapter;
|
||||
}
|
||||
@@ -182,7 +173,14 @@ public class Application {
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
TopicUtils.ensureTopicCreated(this.zkConnect, this.topic, 1, 1);
|
||||
ZkUtils zkUtils = new ZkUtils(new ZkClient(this.zkConnect, 6000, 6000,
|
||||
ZKStringSerializer$.MODULE$), null, false);
|
||||
try {
|
||||
AdminUtils.createTopic(zkUtils, topic, 1, 1, new Properties());
|
||||
}
|
||||
catch (TopicExistsException e) {
|
||||
// no-op
|
||||
}
|
||||
this.running = true;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user