diff --git a/spring-kafka/src/main/java/org/springframework/kafka/core/DefaultKafkaConsumerFactory.java b/spring-kafka/src/main/java/org/springframework/kafka/core/DefaultKafkaConsumerFactory.java index dfee6ec1..466659b3 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/core/DefaultKafkaConsumerFactory.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/core/DefaultKafkaConsumerFactory.java @@ -22,28 +22,55 @@ import java.util.Map; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.serialization.Deserializer; /** * The {@link ConsumerFactory} implementation to produce a new {@link Consumer} instance - * for provided {@link Map} {@code configs} on each {@link #createConsumer()} + * for provided {@link Map} {@code configs} and optional {@link Deserializer} {@code keyDeserializer}, + * {@code valueDeserializer} implementations on each {@link #createConsumer()} * invocation. * * @param the key type. * @param the value type. * * @author Gary Russell + * @author Murali Reddy */ public class DefaultKafkaConsumerFactory implements ConsumerFactory { private final Map configs; + private Deserializer keyDeserializer; + + private Deserializer valueDeserializer; + public DefaultKafkaConsumerFactory(Map configs) { + this(configs, null, null); + } + + public DefaultKafkaConsumerFactory(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { this.configs = new HashMap<>(configs); + this.keyDeserializer = keyDeserializer; + this.valueDeserializer = valueDeserializer; + } + + public void setKeyDeserializer(Deserializer keyDeserializer) { + this.keyDeserializer = keyDeserializer; + } + + public void setValueDeserializer(Deserializer valueDeserializer) { + this.valueDeserializer = valueDeserializer; } @Override public Consumer createConsumer() { - return new KafkaConsumer<>(this.configs); + return createKafkaConsumer(); + } + + protected KafkaConsumer createKafkaConsumer() { + return new KafkaConsumer(this.configs, this.keyDeserializer, this.valueDeserializer); } @Override diff --git a/spring-kafka/src/main/java/org/springframework/kafka/core/DefaultKafkaProducerFactory.java b/spring-kafka/src/main/java/org/springframework/kafka/core/DefaultKafkaProducerFactory.java index 3eaea3fb..b3c97523 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/core/DefaultKafkaProducerFactory.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/core/DefaultKafkaProducerFactory.java @@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; + import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; @@ -32,6 +33,7 @@ import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.serialization.Serializer; import org.springframework.beans.factory.DisposableBean; import org.springframework.context.Lifecycle; @@ -40,6 +42,11 @@ import org.springframework.context.Lifecycle; * The {@link ProducerFactory} implementation for the {@code singleton} shared {@link Producer} * instance. *

+ * This implementation will produce a new {@link Producer} instance + * for provided {@link Map} {@code configs} and optional {@link Serializer} {@code keySerializer}, + * {@code valueSerializer} implementations on each {@link #createProducer()} + * invocation. + *

* The {@link Producer} instance is freed from the external {@link Producer#close()} invocation * with the internal wrapper. The real {@link Producer#close()} is called on the target * {@link Producer} during the {@link Lifecycle#stop()} or {@link DisposableBean#destroy()}. @@ -48,6 +55,7 @@ import org.springframework.context.Lifecycle; * @param the value type. * * @author Gary Russell + * @author Murali Reddy */ public class DefaultKafkaProducerFactory implements ProducerFactory, Lifecycle, DisposableBean { @@ -57,10 +65,29 @@ public class DefaultKafkaProducerFactory implements ProducerFactory, private volatile CloseSafeProducer producer; + private Serializer keySerializer; + + private Serializer valueSerializer; + private volatile boolean running; public DefaultKafkaProducerFactory(Map configs) { + this(configs, null, null); + } + + public DefaultKafkaProducerFactory(Map configs, Serializer keySerializer, + Serializer valueSerializer) { this.configs = new HashMap<>(configs); + this.keySerializer = keySerializer; + this.valueSerializer = valueSerializer; + } + + public void setKeySerializer(Serializer keySerializer) { + this.keySerializer = keySerializer; + } + + public void setValueSerializer(Serializer valueSerializer) { + this.valueSerializer = valueSerializer; } @Override @@ -100,13 +127,17 @@ public class DefaultKafkaProducerFactory implements ProducerFactory, if (this.producer == null) { synchronized (this) { if (this.producer == null) { - this.producer = new CloseSafeProducer(new KafkaProducer(this.configs)); + this.producer = new CloseSafeProducer(createKafkaProducer()); } } } return this.producer; } + protected KafkaProducer createKafkaProducer() { + return new KafkaProducer(this.configs, this.keySerializer, this.valueSerializer); + } + private static class CloseSafeProducer implements Producer { private final Producer delegate; diff --git a/spring-kafka/src/test/java/org/springframework/kafka/annotation/EnableKafkaIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/annotation/EnableKafkaIntegrationTests.java index d77c3b9c..52bd25b6 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/annotation/EnableKafkaIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/annotation/EnableKafkaIntegrationTests.java @@ -187,7 +187,7 @@ public class EnableKafkaIntegrationTests { @Bean public KafkaListenerContainerFactory> - kafkaListenerContainerFactory() { + kafkaListenerContainerFactory() { SimpleKafkaListenerContainerFactory factory = new SimpleKafkaListenerContainerFactory<>(); factory.setConsumerFactory(consumerFactory()); return factory; @@ -195,7 +195,7 @@ public class EnableKafkaIntegrationTests { @Bean public KafkaListenerContainerFactory> - kafkaJsonListenerContainerFactory() { + kafkaJsonListenerContainerFactory() { SimpleKafkaListenerContainerFactory factory = new SimpleKafkaListenerContainerFactory<>(); factory.setConsumerFactory(consumerFactory()); factory.setMessageConverter(new StringJsonMessageConverter()); @@ -204,7 +204,7 @@ public class EnableKafkaIntegrationTests { @Bean public KafkaListenerContainerFactory> - kafkaManualAckListenerContainerFactory() { + kafkaManualAckListenerContainerFactory() { SimpleKafkaListenerContainerFactory factory = new SimpleKafkaListenerContainerFactory<>(); factory.setConsumerFactory(manualConsumerFactory()); factory.setAckMode(AckMode.MANUAL_IMMEDIATE); @@ -213,7 +213,7 @@ public class EnableKafkaIntegrationTests { @Bean public KafkaListenerContainerFactory> - kafkaAutoStartFalseListenerContainerFactory() { + kafkaAutoStartFalseListenerContainerFactory() { SimpleKafkaListenerContainerFactory factory = new SimpleKafkaListenerContainerFactory<>(); factory.setConsumerFactory(consumerFactory()); factory.setAutoStartup(false); @@ -222,7 +222,7 @@ public class EnableKafkaIntegrationTests { @Bean public KafkaListenerContainerFactory> - kafkaRebalanceListenerContainerFactory() { + kafkaRebalanceListenerContainerFactory() { SimpleKafkaListenerContainerFactory factory = new SimpleKafkaListenerContainerFactory<>(); factory.setConsumerFactory(consumerFactory()); factory.setConsumerRebalanceListener(consumerRebalanceListener()); @@ -366,9 +366,9 @@ public class EnableKafkaIntegrationTests { } @KafkaListener(id = "fiz", topicPartitions = { - @TopicPartition(topic = "annotated5", partitions = {"0", "1"}), - @TopicPartition(topic = "annotated6", partitions = {"0", "1"}) - }) + @TopicPartition(topic = "annotated5", partitions = { "0", "1" }), + @TopicPartition(topic = "annotated6", partitions = { "0", "1" }) + }) public void listen5(ConsumerRecord record) { this.record = record; this.latch5.countDown(); diff --git a/spring-kafka/src/test/java/org/springframework/kafka/listener/ConcurrentMessageListenerContainerTests.java b/spring-kafka/src/test/java/org/springframework/kafka/listener/ConcurrentMessageListenerContainerTests.java index 446398a5..06c1e17e 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/listener/ConcurrentMessageListenerContainerTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/listener/ConcurrentMessageListenerContainerTests.java @@ -172,6 +172,60 @@ public class ConcurrentMessageListenerContainerTests { logger.info("Stop auto"); } + @Test + public void testAutoCommitWithRebalanceListener() throws Exception { + logger.info("Start auto"); + Map props = KafkaTestUtils.consumerProps("test10", "true", embeddedKafka); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory(props); + ConcurrentMessageListenerContainer container = + new ConcurrentMessageListenerContainer<>(cf, topic1); + final CountDownLatch latch = new CountDownLatch(4); + container.setMessageListener(new MessageListener() { + + @Override + public void onMessage(ConsumerRecord message) { + logger.info("auto: " + message); + latch.countDown(); + } + }); + final CountDownLatch rebalancePartitionsAssignedLatch = new CountDownLatch(2); + final CountDownLatch rebalancePartitionsRevokedLatch = new CountDownLatch(2); + container.setConsumerRebalanceListener(new ConsumerRebalanceListener() { + + @Override + public void onPartitionsRevoked(Collection partitions) { + logger.info("In test, partitions revoked:" + partitions); + rebalancePartitionsRevokedLatch.countDown(); + } + + @Override + public void onPartitionsAssigned(Collection partitions) { + logger.info("In test, partitions assigned:" + partitions); + rebalancePartitionsAssignedLatch.countDown(); + } + + }); + + container.setConcurrency(2); + container.setBeanName("testAuto"); + container.start(); + ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()); + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + ProducerFactory pf = new DefaultKafkaProducerFactory(senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf); + template.setDefaultTopic(topic1); + template.send(0, "foo"); + template.send(2, "bar"); + template.send(0, "baz"); + template.send(2, "qux"); + template.flush(); + assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue(); + assertThat(rebalancePartitionsAssignedLatch.await(60, TimeUnit.SECONDS)).isTrue(); + assertThat(rebalancePartitionsRevokedLatch.await(60, TimeUnit.SECONDS)).isTrue(); + container.stop(); + logger.info("Stop auto"); + } + @Test public void testAfterListenCommit() throws Exception { logger.info("Start manual");