Kafka Outbound Endpoints - Add futuresChannel

Currently, the only way to block is to set `sync=true` which waits for the future.
The problem with this is it only supports one-at-a-time publication.

Add an option to send the send futures to a channel, allowing the application to
send multiple records and then wait on the futures later.

Also fix long-running test.

* Add docs.
This commit is contained in:
Gary Russell
2020-10-16 14:37:37 -04:00
committed by GitHub
parent 4027adc392
commit 9c6000e453
7 changed files with 194 additions and 4 deletions

View File

@@ -22,10 +22,13 @@ import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.mock;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeoutException;
import org.apache.kafka.clients.producer.MockProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.IntegerSerializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.junit.jupiter.api.Test;
@@ -109,12 +112,16 @@ class KafkaOutboundAdapterParserTests {
@SuppressWarnings("unchecked")
ProducerFactory<Integer, String> pf = mock(ProducerFactory.class);
given(pf.createProducer()).willReturn(mockProducer);
Map<String, Object> props = new HashMap<>();
props.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 10);
given(pf.getConfigurationProperties()).willReturn(props);
KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
KafkaProducerMessageHandler<Integer, String> handler = new KafkaProducerMessageHandler<>(template);
handler.setBeanFactory(mock(BeanFactory.class));
handler.afterPropertiesSet();
handler.setSync(true);
handler.setSendTimeout(10);
handler.setTopicExpression(new LiteralExpression("foo"));
Executors.newSingleThreadExecutor()

View File

@@ -23,6 +23,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
@@ -49,6 +50,7 @@ import org.springframework.integration.kafka.channel.PollableKafkaChannel;
import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter;
import org.springframework.integration.kafka.inbound.KafkaMessageSource;
import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler;
import org.springframework.integration.kafka.support.KafkaIntegrationHeaders;
import org.springframework.integration.kafka.support.RawRecordHeaderErrorMessageStrategy;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.integration.test.util.TestUtils;
@@ -137,6 +139,9 @@ public class KafkaDslTests {
@Autowired
private PollableChannel errorChannel;
@Autowired
private PollableChannel futuresChannel;
@Autowired(required = false)
@Qualifier("topic1ListenerContainer")
private MessageListenerContainer messageListenerContainer;
@@ -164,6 +169,12 @@ public class KafkaDslTests {
assertThat(TestUtils.getPropertyValue(this.kafkaProducer1, "headerMapper")).isSameAs(this.mapper);
for (int i = 0; i < 200; i++) {
Message<?> future = this.futuresChannel.receive(10000);
assertThat(future).isNotNull();
((Future<?>) future.getPayload()).get(10, TimeUnit.SECONDS);
}
for (int i = 0; i < 100; i++) {
Message<?> receive = this.listeningFromKafkaResults1.receive(20000);
assertThat(receive).isNotNull();
@@ -327,10 +338,16 @@ public class KafkaDslTests {
return new DefaultKafkaProducerFactory<>(props);
}
@Bean
public PollableChannel futuresChannel() {
return new QueueChannel();
}
@Bean
public IntegrationFlow sendToKafkaFlow() {
return f -> f
.<String>split(p -> Stream.generate(() -> p).limit(101).iterator(), null)
.enrichHeaders(h -> h.header(KafkaIntegrationHeaders.FUTURE_TOKEN, "foo"))
.publishSubscribeChannel(c -> c
.subscribe(sf -> sf.handle(
kafkaMessageHandler(producerFactory(), TEST_TOPIC1)
@@ -354,6 +371,7 @@ public class KafkaDslTests {
return Kafka
.outboundChannelAdapter(producerFactory)
.futuresChannel("futuresChannel")
.sync(true)
.messageKey(m -> m
.getHeaders()