Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use ConsumerProperties in KafkaMessageSource

Browse Source 
* Preserve existing constructors

* Add @deprecated in java docs
This commit is contained in:
Anshul Mehra
2019-08-23 14:07:44 -04:00
committed by Artem Bilan
parent e6cdb8818f
commit e52c535f7c
11 changed files with 375 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/AllXmlTests-context.xml
View File

@@ -29,11 +29,15 @@
</constructor-arg>
</bean>
<bean id="cp" class="org.springframework.kafka.listener.ConsumerProperties">
<constructor-arg name="topics" value="one"/>
<property name="groupId" value="groupForTopic1"/>
</bean>
<int-kafka:inbound-channel-adapter
channel="fromOne"
consumer-factory="cf"
topics="one"
group-id="groupForTopic1">
consumer-properties="cp">
<int:poller fixed-delay="5000"/>
</int-kafka:inbound-channel-adapter>

22
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundChannelAdapterParserTests-context.xml
View File

@@ -10,25 +10,21 @@
<int-kafka:inbound-channel-adapter
id="adapter1"
consumer-factory="consumerFactory"
consumer-properties="consumerProperties1"
ack-factory="ackFactory"
topics="topic1"
channel="inbound"
client-id="client"
group-id="group"
message-converter="converter"
payload-type="java.lang.String"
raw-header="true"
auto-startup="false"
rebalance-listener="rebal">
auto-startup="false">
<int:poller fixed-delay="5000"/>
</int-kafka:inbound-channel-adapter>
<int-kafka:inbound-channel-adapter
id="adapter2"
consumer-factory="multiFetchConsumerFactory"
consumer-properties="consumerProperties2"
allow-multi-fetch="true"
topics="topic1, topic2"
group-id="group"
auto-startup="false"
channel="inbound">
<int:poller fixed-delay="5000"/>
@@ -42,6 +38,18 @@
</constructor-arg>
</bean>
<bean id="consumerProperties1" class="org.springframework.kafka.listener.ConsumerProperties">
<constructor-arg name="topics" value="topic1"/>
<property name="groupId" value="group"/>
<property name="clientId" value="client"/>
<property name="consumerRebalanceListener" ref="rebal"/>
</bean>
<bean id="consumerProperties2" class="org.springframework.kafka.listener.ConsumerProperties">
<constructor-arg name="topics" value="topic1, topic2"/>
<property name="groupId" value="group"/>
</bean>
<bean id="multiFetchConsumerFactory" class="org.springframework.kafka.core.DefaultKafkaConsumerFactory">
<constructor-arg>
<map>

4
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/dsl/KafkaDslTests.java
View File

@@ -57,6 +57,7 @@ import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaProducerFactory;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.core.ProducerFactory;
import org.springframework.kafka.listener.ConsumerProperties;
import org.springframework.kafka.listener.ContainerProperties;
import org.springframework.kafka.listener.ContainerProperties.AckMode;
import org.springframework.kafka.listener.GenericMessageListenerContainer;
@@ -83,6 +84,7 @@ import org.springframework.test.context.junit4.SpringRunner;
* @author Nasko Vasilev
* @author Biju Kunjummen
* @author Gary Russell
* @author Anshul Mehra
*
* @since 3.0
*/
@@ -337,7 +339,7 @@ public class KafkaDslTests {
@Bean
public IntegrationFlow sourceFlow() {
return IntegrationFlows
.from(Kafka.inboundChannelAdapter(consumerFactory(), TEST_TOPIC3),
.from(Kafka.inboundChannelAdapter(consumerFactory(), new ConsumerProperties(TEST_TOPIC3)),
e -> e.poller(Pollers.fixedDelay(100)))
.handle(p -> {
this.fromSource = p.getPayload();

7
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/inbound/MessageSourceIntegrationTests.java
View File

@@ -32,6 +32,7 @@ import org.junit.Test;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaProducerFactory;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.listener.ConsumerProperties;
import org.springframework.kafka.test.EmbeddedKafkaBroker;
import org.springframework.kafka.test.rule.EmbeddedKafkaRule;
import org.springframework.kafka.test.utils.KafkaTestUtils;
@@ -40,6 +41,7 @@ import org.springframework.messaging.Message;
/**
* @author Gary Russell
* @author Artem Bilan
* @author Anshul Mehra
*
* @since 3.0.1
*
@@ -59,9 +61,9 @@ public class MessageSourceIntegrationTests {
consumerProps.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 2);
consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
DefaultKafkaConsumerFactory<Integer, String> consumerFactory = new DefaultKafkaConsumerFactory<>(consumerProps);
KafkaMessageSource<Integer, String> source = new KafkaMessageSource<>(consumerFactory, TOPIC1);
ConsumerProperties consumerProperties = new ConsumerProperties(TOPIC1);
final CountDownLatch assigned = new CountDownLatch(1);
source.setRebalanceListener(new ConsumerRebalanceListener() {
consumerProperties.setConsumerRebalanceListener(new ConsumerRebalanceListener() {
@Override
public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
@@ -73,6 +75,7 @@ public class MessageSourceIntegrationTests {
}
});
KafkaMessageSource<Integer, String> source = new KafkaMessageSource<>(consumerFactory, consumerProperties);
Map<String, Object> producerProps = KafkaTestUtils.producerProps(embeddedKafka);
DefaultKafkaProducerFactory<Object, Object> producerFactory = new DefaultKafkaProducerFactory<>(producerProps);

32
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/inbound/MessageSourceTests.java
View File

@@ -62,6 +62,7 @@ import org.springframework.integration.acks.AcknowledgmentCallback;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.listener.ConsumerProperties;
import org.springframework.kafka.support.KafkaHeaders;
import org.springframework.kafka.test.utils.KafkaTestUtils;
import org.springframework.messaging.Message;
@@ -111,7 +112,7 @@ public class MessageSourceTests {
willReturn(Collections.singletonMap(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1)).given(consumerFactory)
.getConfigurationProperties();
given(consumerFactory.createConsumer(isNull(), anyString(), isNull())).willReturn(consumer);
KafkaMessageSource source = new KafkaMessageSource(consumerFactory, "foo");
KafkaMessageSource source = new KafkaMessageSource(consumerFactory, new ConsumerProperties("foo"));
source.setRawMessageHeader(true);
Message<?> received = source.receive();
@@ -205,7 +206,7 @@ public class MessageSourceTests {
willReturn(Collections.singletonMap(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1)).given(consumerFactory)
.getConfigurationProperties();
given(consumerFactory.createConsumer(isNull(), anyString(), isNull())).willReturn(consumer);
KafkaMessageSource source = new KafkaMessageSource(consumerFactory, "foo");
KafkaMessageSource source = new KafkaMessageSource(consumerFactory, new ConsumerProperties("foo"));
Message<?> received1 = source.receive();
consumer.paused(); // need some other interaction with mock between polls for InOrder
@@ -279,8 +280,9 @@ public class MessageSourceTests {
willReturn(Collections.singletonMap(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1)).given(consumerFactory)
.getConfigurationProperties();
given(consumerFactory.createConsumer(isNull(), anyString(), isNull())).willReturn(consumer);
KafkaMessageSource source = new KafkaMessageSource(consumerFactory, "foo");
source.setCommitTimeout(Duration.ofSeconds(30));
ConsumerProperties consumerProperties = new ConsumerProperties("foo");
consumerProperties.setSyncCommitTimeout(Duration.ofSeconds(30));
KafkaMessageSource source = new KafkaMessageSource(consumerFactory, consumerProperties);
Message<?> received = source.receive();
assertThat(received.getHeaders().get(KafkaHeaders.OFFSET)).isEqualTo(0L);
@@ -347,7 +349,7 @@ public class MessageSourceTests {
willReturn(Collections.singletonMap(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1)).given(consumerFactory)
.getConfigurationProperties();
given(consumerFactory.createConsumer(isNull(), anyString(), isNull())).willReturn(consumer);
KafkaMessageSource source = new KafkaMessageSource(consumerFactory, "foo");
KafkaMessageSource source = new KafkaMessageSource(consumerFactory, new ConsumerProperties("foo"));
Message<?> received1 = source.receive();
consumer.paused(); // need some other interaction with mock between polls for InOrder
@@ -407,17 +409,17 @@ public class MessageSourceTests {
@Test
public void testMaxPollRecords() {
KafkaMessageSource source = new KafkaMessageSource(new DefaultKafkaConsumerFactory<>(Collections.emptyMap()),
"topic");
new ConsumerProperties("topic"));
assertThat((TestUtils.getPropertyValue(source, "consumerFactory.configs", Map.class)
.get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG))).isEqualTo(1);
source = new KafkaMessageSource(new DefaultKafkaConsumerFactory<>(
Collections.singletonMap(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 2)), "topic");
Collections.singletonMap(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 2)), new ConsumerProperties("topic"));
assertThat((TestUtils.getPropertyValue(source, "consumerFactory.configs", Map.class)
.get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG))).isEqualTo(1);
try {
new KafkaMessageSource((new DefaultKafkaConsumerFactory(Collections.emptyMap()) {
}), "topic");
}), new ConsumerProperties("topic"));
fail("Expected exception");
}
catch (IllegalArgumentException e) {
@@ -441,17 +443,17 @@ public class MessageSourceTests {
records1.put(topicPartition, Arrays.asList(
new ConsumerRecord("foo", 0, 0L, 0L, TimestampType.NO_TIMESTAMP_TYPE, 0, 0, 0, null, "foo")));
ConsumerRecords cr1 = new ConsumerRecords(records1);
given(consumer.poll(Duration.of(2, ChronoUnit.SECONDS))).willReturn(cr1, ConsumerRecords.EMPTY);
given(consumer.poll(Duration.of(20 * 5000, ChronoUnit.MILLIS))).willReturn(cr1, ConsumerRecords.EMPTY);
Map<TopicPartition, List<ConsumerRecord>> records2 = new LinkedHashMap<>();
records2.put(topicPartition, Arrays.asList(
new ConsumerRecord("foo", 0, 1L, 0L, TimestampType.NO_TIMESTAMP_TYPE, 0, 0, 0, null, "foo")));
ConsumerRecords cr2 = new ConsumerRecords(records2);
given(consumer.poll(Duration.of(50, ChronoUnit.MILLIS))).willReturn(cr2, ConsumerRecords.EMPTY);
given(consumer.poll(Duration.of(5000, ChronoUnit.MILLIS))).willReturn(cr2, ConsumerRecords.EMPTY);
ConsumerFactory consumerFactory = mock(ConsumerFactory.class);
willReturn(Collections.singletonMap(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1)).given(consumerFactory)
.getConfigurationProperties();
given(consumerFactory.createConsumer(isNull(), anyString(), isNull())).willReturn(consumer);
KafkaMessageSource source = new KafkaMessageSource(consumerFactory, "foo");
KafkaMessageSource source = new KafkaMessageSource(consumerFactory, new ConsumerProperties("foo"));
source.setRawMessageHeader(true);
Message<?> received = source.receive();
@@ -477,13 +479,13 @@ public class MessageSourceTests {
InOrder inOrder = inOrder(consumer);
inOrder.verify(consumer).subscribe(anyCollection(), any(ConsumerRebalanceListener.class));
// assignTimeout used on initial poll (before partition assigned)
inOrder.verify(consumer).poll(Duration.of(2, ChronoUnit.SECONDS));
inOrder.verify(consumer).poll(Duration.of(20 * 5000, ChronoUnit.MILLIS));
inOrder.verify(consumer).commitSync(Collections.singletonMap(topicPartition, new OffsetAndMetadata(1L)));
// pollTimeout used on subsequent polls
inOrder.verify(consumer).poll(Duration.of(50, ChronoUnit.MILLIS));
inOrder.verify(consumer).poll(Duration.of(5000, ChronoUnit.MILLIS));
inOrder.verify(consumer).commitSync(Collections.singletonMap(topicPartition, new OffsetAndMetadata(2L)));
// assignTimeout used after partitions revoked
inOrder.verify(consumer).poll(Duration.of(2, ChronoUnit.SECONDS));
inOrder.verify(consumer).poll(Duration.of(20 * 5000, ChronoUnit.MILLIS));
}
@SuppressWarnings({ "rawtypes", "unchecked" })
@@ -513,7 +515,7 @@ public class MessageSourceTests {
willReturn(Collections.singletonMap(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 4)).given(consumerFactory)
.getConfigurationProperties();
given(consumerFactory.createConsumer(isNull(), anyString(), isNull())).willReturn(consumer);
KafkaMessageSource source = new KafkaMessageSource(consumerFactory, true, "foo");
KafkaMessageSource source = new KafkaMessageSource(consumerFactory, new ConsumerProperties("foo"), true);
source.setRawMessageHeader(true);
Message<?> received = source.receive();

7
spring-integration-kafka/src/test/kotlin/org/springframework/integration/kafka/dsl/KafkaDslKotlinTests.kt
View File

@@ -17,15 +17,12 @@
package org.springframework.integration.kafka.dsl
import assertk.assertThat
import assertk.assertions.contains
import assertk.assertions.isEqualTo
import assertk.assertions.isInstanceOf
import assertk.assertions.isNotNull
import assertk.assertions.isNull
import assertk.assertions.isSameAs
import assertk.assertions.isTrue
import assertk.catch
import kafka.tools.ConsoleProducer
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener
import org.apache.kafka.clients.producer.ProducerConfig
@@ -44,7 +41,6 @@ import org.springframework.integration.config.EnableIntegration
import org.springframework.integration.dsl.IntegrationFlow
import org.springframework.integration.dsl.IntegrationFlows
import org.springframework.integration.dsl.Pollers
import org.springframework.integration.expression.ValueExpression
import org.springframework.integration.handler.advice.ErrorMessageSendingRecoverer
import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter
import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler
@@ -57,6 +53,7 @@ import org.springframework.kafka.core.DefaultKafkaConsumerFactory
import org.springframework.kafka.core.DefaultKafkaProducerFactory
import org.springframework.kafka.core.KafkaTemplate
import org.springframework.kafka.core.ProducerFactory
import org.springframework.kafka.listener.ConsumerProperties
import org.springframework.kafka.listener.ContainerProperties
import org.springframework.kafka.listener.GenericMessageListenerContainer
import org.springframework.kafka.listener.KafkaMessageListenerContainer
@@ -320,7 +317,7 @@ class KafkaDslKotlinTests {
@Bean
fun sourceFlow() =
IntegrationFlows
.from(Kafka.inboundChannelAdapter(consumerFactory(), TEST_TOPIC3)) { e -> e.poller(Pollers.fixedDelay(100)) }
.from(Kafka.inboundChannelAdapter(consumerFactory(), ConsumerProperties(TEST_TOPIC3))) { e -> e.poller(Pollers.fixedDelay(100)) }
.handle({ p ->
this.fromSource = p.getPayload()
this.sourceFlowLatch.countDown()