Kafka Outbound Endpoints - Add futuresChannel
Currently, the only way to block is to set `sync=true` which waits for the future. The problem with this is it only supports one-at-a-time publication. Add an option to send the send futures to a channel, allowing the application to send multiple records and then wait on the futures later. Also fix long-running test. * Add docs.
This commit is contained in:
@@ -22,10 +22,13 @@ import static org.mockito.BDDMockito.given;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import org.apache.kafka.clients.producer.MockProducer;
|
||||
import org.apache.kafka.clients.producer.ProducerConfig;
|
||||
import org.apache.kafka.common.serialization.IntegerSerializer;
|
||||
import org.apache.kafka.common.serialization.StringSerializer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -109,12 +112,16 @@ class KafkaOutboundAdapterParserTests {
|
||||
@SuppressWarnings("unchecked")
|
||||
ProducerFactory<Integer, String> pf = mock(ProducerFactory.class);
|
||||
given(pf.createProducer()).willReturn(mockProducer);
|
||||
Map<String, Object> props = new HashMap<>();
|
||||
props.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 10);
|
||||
given(pf.getConfigurationProperties()).willReturn(props);
|
||||
KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
|
||||
KafkaProducerMessageHandler<Integer, String> handler = new KafkaProducerMessageHandler<>(template);
|
||||
handler.setBeanFactory(mock(BeanFactory.class));
|
||||
handler.afterPropertiesSet();
|
||||
|
||||
handler.setSync(true);
|
||||
handler.setSendTimeout(10);
|
||||
handler.setTopicExpression(new LiteralExpression("foo"));
|
||||
|
||||
Executors.newSingleThreadExecutor()
|
||||
|
||||
@@ -23,6 +23,7 @@ import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@@ -49,6 +50,7 @@ import org.springframework.integration.kafka.channel.PollableKafkaChannel;
|
||||
import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter;
|
||||
import org.springframework.integration.kafka.inbound.KafkaMessageSource;
|
||||
import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler;
|
||||
import org.springframework.integration.kafka.support.KafkaIntegrationHeaders;
|
||||
import org.springframework.integration.kafka.support.RawRecordHeaderErrorMessageStrategy;
|
||||
import org.springframework.integration.support.MessageBuilder;
|
||||
import org.springframework.integration.test.util.TestUtils;
|
||||
@@ -137,6 +139,9 @@ public class KafkaDslTests {
|
||||
@Autowired
|
||||
private PollableChannel errorChannel;
|
||||
|
||||
@Autowired
|
||||
private PollableChannel futuresChannel;
|
||||
|
||||
@Autowired(required = false)
|
||||
@Qualifier("topic1ListenerContainer")
|
||||
private MessageListenerContainer messageListenerContainer;
|
||||
@@ -164,6 +169,12 @@ public class KafkaDslTests {
|
||||
|
||||
assertThat(TestUtils.getPropertyValue(this.kafkaProducer1, "headerMapper")).isSameAs(this.mapper);
|
||||
|
||||
for (int i = 0; i < 200; i++) {
|
||||
Message<?> future = this.futuresChannel.receive(10000);
|
||||
assertThat(future).isNotNull();
|
||||
((Future<?>) future.getPayload()).get(10, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
for (int i = 0; i < 100; i++) {
|
||||
Message<?> receive = this.listeningFromKafkaResults1.receive(20000);
|
||||
assertThat(receive).isNotNull();
|
||||
@@ -327,10 +338,16 @@ public class KafkaDslTests {
|
||||
return new DefaultKafkaProducerFactory<>(props);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public PollableChannel futuresChannel() {
|
||||
return new QueueChannel();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public IntegrationFlow sendToKafkaFlow() {
|
||||
return f -> f
|
||||
.<String>split(p -> Stream.generate(() -> p).limit(101).iterator(), null)
|
||||
.enrichHeaders(h -> h.header(KafkaIntegrationHeaders.FUTURE_TOKEN, "foo"))
|
||||
.publishSubscribeChannel(c -> c
|
||||
.subscribe(sf -> sf.handle(
|
||||
kafkaMessageHandler(producerFactory(), TEST_TOPIC1)
|
||||
@@ -354,6 +371,7 @@ public class KafkaDslTests {
|
||||
|
||||
return Kafka
|
||||
.outboundChannelAdapter(producerFactory)
|
||||
.futuresChannel("futuresChannel")
|
||||
.sync(true)
|
||||
.messageKey(m -> m
|
||||
.getHeaders()
|
||||
|
||||
Reference in New Issue
Block a user