From c32be995f6169cf704ab4ecdd7c421552e1158ac Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Mon, 8 Nov 2021 19:21:49 -0500 Subject: [PATCH 01/19] 4.0.x changes for Kafka Streams tests Migrating StreamListener based Kafka Streams binder tests to use the funcitonal model --- ...reamsInteractiveQueryIntegrationTests.java | 19 +- ...serializationErrorHandlerByKafkaTests.java | 271 ----------------- ...serializtionErrorHandlerByBinderTests.java | 286 ------------------ ...afkaStreamsBinderHealthIndicatorTests.java | 48 ++- ...aStreamsBinderMultipleInputTopicsTest.java | 22 +- ...rPojoInputAndPrimitiveTypeOutputTests.java | 21 +- ...fkaStreamsNativeEncodingDecodingTests.java | 186 ------------ ...afkaStreamsStateStoreIntegrationTests.java | 168 +++------- ...PojoInputStringOutputIntegrationTests.java | 19 +- ...ProcessorsWithSameNameAndBindingTests.java | 95 ------ .../PerRecordAvroContentTypeTests.java | 184 ----------- .../integration/utils/TestAvroSerializer.java | 63 ---- .../binder/kstream/integTest-1.properties | 6 - 13 files changed, 85 insertions(+), 1303 deletions(-) delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializationErrorHandlerByKafkaTests.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializtionErrorHandlerByBinderTests.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsNativeEncodingDecodingTests.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/MultiProcessorsWithSameNameAndBindingTests.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/PerRecordAvroContentTypeTests.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/utils/TestAvroSerializer.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/test/resources/org/springframework/cloud/stream/binder/kstream/integTest-1.properties diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java index 42121993f..ab8fa087e 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java @@ -18,6 +18,7 @@ package org.springframework.cloud.stream.binder.kafka.streams; import java.util.List; import java.util.Map; +import java.util.function.Function; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -38,7 +39,6 @@ import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Test; import org.mockito.Mockito; @@ -46,9 +46,6 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.kafka.KafkaProperties; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; @@ -61,7 +58,6 @@ import org.springframework.kafka.support.serializer.JsonSerde; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.internal.verification.VerificationModeFactory.times; @@ -125,12 +121,13 @@ public class KafkaStreamsInteractiveQueryIntegrationTests { } @Test - @Ignore - public void testKstreamBinderWithPojoInputAndStringOuput() throws Exception { + public void testKstreamBinderWithPojoInputAndStringOuput() { SpringApplication app = new SpringApplication(ProductCountApplication.class); app.setWebApplicationType(WebApplicationType.NONE); ConfigurableApplicationContext context = app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input", + "--spring.cloud.stream.function.bindings.process-out-0=output", "--spring.cloud.stream.bindings.input.destination=foos", "--spring.cloud.stream.bindings.output.destination=counts-id", "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", @@ -203,15 +200,13 @@ public class KafkaStreamsInteractiveQueryIntegrationTests { } - @EnableBinding(KafkaStreamsProcessor.class) @EnableAutoConfiguration public static class ProductCountApplication { - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { + @Bean + public Function, KStream> process() { - return input.filter((key, product) -> product.getId() == 123) + return input -> input.filter((key, product) -> product.getId() == 123) .map((key, value) -> new KeyValue<>(value.id, value)) .groupByKey(Grouped.with(new Serdes.IntegerSerde(), new JsonSerde<>(Product.class))) diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializationErrorHandlerByKafkaTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializationErrorHandlerByKafkaTests.java deleted file mode 100644 index a2323c555..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializationErrorHandlerByKafkaTests.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.integration; - -import java.time.Duration; -import java.util.Arrays; -import java.util.Map; - -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.kstream.Grouped; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.Materialized; -import org.apache.kafka.streams.kstream.TimeWindows; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; - -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; -import org.springframework.cloud.stream.binder.kafka.utils.DlqPartitionFunction; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.PropertySource; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.test.EmbeddedKafkaBroker; -import org.springframework.kafka.test.rule.EmbeddedKafkaRule; -import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; - -/** - * @author Soby Chacko - */ -@RunWith(SpringRunner.class) -@ContextConfiguration -@DirtiesContext -public abstract class DeserializationErrorHandlerByKafkaTests { - - @ClassRule - public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, - "abc-DeserializationErrorHandlerByKafkaTests-In", - "xyz-DeserializationErrorHandlerByKafkaTests-In", - "DeserializationErrorHandlerByKafkaTests-out", - "error.abc-DeserializationErrorHandlerByKafkaTests-In.group", - "error.xyz-DeserializationErrorHandlerByKafkaTests-In.group", - "error.word1.groupx", - "error.word2.groupx"); - - private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule - .getEmbeddedKafka(); - - @SpyBean - org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsMessageConversionDelegate conversionDelegate; - - private static Consumer consumer; - - @BeforeClass - public static void setUp() { - System.setProperty("spring.cloud.stream.kafka.streams.binder.brokers", - embeddedKafka.getBrokersAsString()); - - System.setProperty("server.port", "0"); - System.setProperty("spring.jmx.enabled", "false"); - - Map consumerProps = KafkaTestUtils.consumerProps("fooc", "false", - embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - consumer = cf.createConsumer(); - embeddedKafka.consumeFromEmbeddedTopics(consumer, "DeserializationErrorHandlerByKafkaTests-out", "DeserializationErrorHandlerByKafkaTests-out"); - } - - @AfterClass - public static void tearDown() { - consumer.close(); - System.clearProperty("spring.cloud.stream.kafka.streams.binder.brokers"); - System.clearProperty("server.port"); - System.clearProperty("spring.jmx.enabled"); - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.destination=abc-DeserializationErrorHandlerByKafkaTests-In", - "spring.cloud.stream.bindings.output.destination=DeserializationErrorHandlerByKafkaTests-Out", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=deser-kafka-dlq", - "spring.cloud.stream.bindings.input.group=group", - "spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler=sendToDlq", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.valueSerde=" - + "org.apache.kafka.common.serialization.Serdes$IntegerSerde" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - public static class DeserializationByKafkaAndDlqTests - extends DeserializationErrorHandlerByKafkaTests { - - @Test - @Ignore - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("abc-DeserializationErrorHandlerByKafkaTests-In"); - template.sendDefault(1, null, "foobar"); - - Map consumerProps = KafkaTestUtils.consumerProps("foobar", - "false", embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - Consumer consumer1 = cf.createConsumer(); - embeddedKafka.consumeFromAnEmbeddedTopic(consumer1, "error.abc-DeserializationErrorHandlerByKafkaTests-In.group"); - - ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer1, - "error.abc-DeserializationErrorHandlerByKafkaTests-In.group"); - assertThat(cr.value()).isEqualTo("foobar"); - assertThat(cr.partition()).isEqualTo(0); // custom partition function - - // Ensuring that the deserialization was indeed done by Kafka natively - verify(conversionDelegate, never()).deserializeOnInbound(any(Class.class), - any(KStream.class)); - verify(conversionDelegate, never()).serializeOnOutbound(any(KStream.class)); - } - - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.destination=xyz-DeserializationErrorHandlerByKafkaTests-In", - "spring.cloud.stream.bindings.output.destination=DeserializationErrorHandlerByKafkaTests-Out", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=deser-kafka-dlq", - "spring.cloud.stream.bindings.input.group=group", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.deserializationExceptionHandler=sendToDlq", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.valueSerde=" - + "org.apache.kafka.common.serialization.Serdes$IntegerSerde" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - public static class DeserializationByKafkaAndDlqPerBindingTests - extends DeserializationErrorHandlerByKafkaTests { - - @Test - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("xyz-DeserializationErrorHandlerByKafkaTests-In"); - template.sendDefault(1, null, "foobar"); - - Map consumerProps = KafkaTestUtils.consumerProps("foobar", - "false", embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - Consumer consumer1 = cf.createConsumer(); - embeddedKafka.consumeFromAnEmbeddedTopic(consumer1, "error.xyz-DeserializationErrorHandlerByKafkaTests-In.group"); - - ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer1, - "error.xyz-DeserializationErrorHandlerByKafkaTests-In.group"); - assertThat(cr.value()).isEqualTo("foobar"); - assertThat(cr.partition()).isEqualTo(0); // custom partition function - - // Ensuring that the deserialization was indeed done by Kafka natively - verify(conversionDelegate, never()).deserializeOnInbound(any(Class.class), - any(KStream.class)); - verify(conversionDelegate, never()).serializeOnOutbound(any(KStream.class)); - } - - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.destination=word1,word2", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=deser-kafka-dlq-multi-input", - "spring.cloud.stream.bindings.input.group=groupx", - "spring.cloud.stream.kafka.streams.binder.serdeError=sendToDlq", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.valueSerde=" - + "org.apache.kafka.common.serialization.Serdes$IntegerSerde" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - // @checkstyle:on - public static class DeserializationByKafkaAndDlqTestsWithMultipleInputs - extends DeserializationErrorHandlerByKafkaTests { - - @Test - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("word1"); - template.sendDefault("foobar"); - - template.setDefaultTopic("word2"); - template.sendDefault("foobar"); - - Map consumerProps = KafkaTestUtils.consumerProps("foobarx", - "false", embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - Consumer consumer1 = cf.createConsumer(); - embeddedKafka.consumeFromEmbeddedTopics(consumer1, "error.word1.groupx", - "error.word2.groupx"); - - ConsumerRecord cr1 = KafkaTestUtils.getSingleRecord(consumer1, - "error.word1.groupx"); - assertThat(cr1.value()).isEqualTo("foobar"); - ConsumerRecord cr2 = KafkaTestUtils.getSingleRecord(consumer1, - "error.word2.groupx"); - assertThat(cr2.value()).isEqualTo("foobar"); - - // Ensuring that the deserialization was indeed done by Kafka natively - verify(conversionDelegate, never()).deserializeOnInbound(any(Class.class), - any(KStream.class)); - verify(conversionDelegate, never()).serializeOnOutbound(any(KStream.class)); - } - - } - - @EnableBinding(KafkaStreamsProcessor.class) - @EnableAutoConfiguration - @PropertySource("classpath:/org/springframework/cloud/stream/binder/kstream/integTest-1.properties") - public static class WordCountProcessorApplication { - - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - - return input - .flatMapValues( - value -> Arrays.asList(value.toLowerCase().split("\\W+"))) - .map((key, value) -> new KeyValue<>(value, value)) - .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) - .windowedBy(TimeWindows.of(Duration.ofMillis(5000))).count(Materialized.as("foo-WordCounts-x")) - .toStream().map((key, value) -> new KeyValue<>(null, - "Count for " + key.key() + " : " + value)); - } - - @Bean - public DlqPartitionFunction partitionFunction() { - return (group, rec, ex) -> 0; - } - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializtionErrorHandlerByBinderTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializtionErrorHandlerByBinderTests.java deleted file mode 100644 index bf2d9cfce..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializtionErrorHandlerByBinderTests.java +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.integration; - -import java.time.Duration; -import java.util.Map; - -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.kstream.Grouped; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.Materialized; -import org.apache.kafka.streams.kstream.TimeWindows; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; - -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.support.serializer.JsonSerde; -import org.springframework.kafka.test.EmbeddedKafkaBroker; -import org.springframework.kafka.test.rule.EmbeddedKafkaRule; -import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; - -/** - * @author Soby Chacko - */ -@RunWith(SpringRunner.class) -@ContextConfiguration -@DirtiesContext -public abstract class DeserializtionErrorHandlerByBinderTests { - - @ClassRule - public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, - "foos", "goos", - "counts-id", "error.foos.foobar-group", "error.goos.foobar-group", "error.foos1.fooz-group", - "error.foos2.fooz-group"); - - private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule - .getEmbeddedKafka(); - - @SpyBean - org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsMessageConversionDelegate conversionDelegate; - - private static Consumer consumer; - - @BeforeClass - public static void setUp() throws Exception { - System.setProperty("spring.cloud.stream.kafka.streams.binder.brokers", - embeddedKafka.getBrokersAsString()); - System.setProperty("server.port", "0"); - System.setProperty("spring.jmx.enabled", "false"); - - Map consumerProps = KafkaTestUtils.consumerProps("kafka-streams-dlq-tests", "false", - embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - consumer = cf.createConsumer(); - embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "counts-id"); - } - - @AfterClass - public static void tearDown() { - consumer.close(); - System.clearProperty("spring.cloud.stream.kafka.streams.binder.brokers"); - System.clearProperty("server.port"); - System.clearProperty("spring.jmx.enabled"); - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", - "spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", - "spring.cloud.stream.bindings.input.destination=foos", - "spring.cloud.stream.bindings.output.destination=counts-id", - "spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", - "spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" - + "=org.apache.kafka.common.serialization.Serdes$IntegerSerde", - "spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" - + "=org.apache.kafka.common.serialization.Serdes$StringSerde", - "spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler=sendToDlq", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id" - + "=deserializationByBinderAndDlqTests", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.dlqPartitions=1", - "spring.cloud.stream.bindings.input.group=foobar-group" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - public static class DeserializationByBinderAndDlqTests - extends DeserializtionErrorHandlerByBinderTests { - - @Test - @Ignore - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("foos"); - template.sendDefault(1, 7, "hello"); - - Map consumerProps = KafkaTestUtils.consumerProps("foobar", - "false", embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - Consumer consumer1 = cf.createConsumer(); - embeddedKafka.consumeFromAnEmbeddedTopic(consumer1, - "error.foos.foobar-group"); - - ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer1, - "error.foos.foobar-group"); - assertThat(cr.value()).isEqualTo("hello"); - assertThat(cr.partition()).isEqualTo(0); - - // Ensuring that the deserialization was indeed done by the binder - verify(conversionDelegate).deserializeOnInbound(any(Class.class), - any(KStream.class)); - } - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", - "spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", - "spring.cloud.stream.bindings.input.destination=goos", - "spring.cloud.stream.bindings.output.destination=counts-id", - "spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", - "spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" - + "=org.apache.kafka.common.serialization.Serdes$IntegerSerde", - "spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" - + "=org.apache.kafka.common.serialization.Serdes$StringSerde", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.deserializationExceptionHandler=sendToDlq", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id" - + "=deserializationByBinderAndDlqTests", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.dlqPartitions=1", - "spring.cloud.stream.bindings.input.group=foobar-group" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - public static class DeserializationByBinderAndDlqSetOnConsumerBindingTests - extends DeserializtionErrorHandlerByBinderTests { - - @Test - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("goos"); - template.sendDefault(1, 7, "hello"); - - Map consumerProps = KafkaTestUtils.consumerProps("foobar", - "false", embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - Consumer consumer1 = cf.createConsumer(); - embeddedKafka.consumeFromAnEmbeddedTopic(consumer1, - "error.goos.foobar-group"); - - ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer1, - "error.goos.foobar-group"); - assertThat(cr.value()).isEqualTo("hello"); - assertThat(cr.partition()).isEqualTo(0); - - // Ensuring that the deserialization was indeed done by the binder - verify(conversionDelegate).deserializeOnInbound(any(Class.class), - any(KStream.class)); - } - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", - "spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", - "spring.cloud.stream.bindings.input.destination=foos1,foos2", - "spring.cloud.stream.bindings.output.destination=counts-id", - "spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", - "spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" - + "=org.apache.kafka.common.serialization.Serdes$StringSerde", - "spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" - + "=org.apache.kafka.common.serialization.Serdes$StringSerde", - "spring.cloud.stream.kafka.streams.binder.serdeError=sendToDlq", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id" - + "=deserializationByBinderAndDlqTestsWithMultipleInputs", - "spring.cloud.stream.bindings.input.group=fooz-group" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - public static class DeserializationByBinderAndDlqTestsWithMultipleInputs - extends DeserializtionErrorHandlerByBinderTests { - - @Test - @SuppressWarnings("unchecked") - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("foos1"); - template.sendDefault("hello"); - - template.setDefaultTopic("foos2"); - template.sendDefault("hello"); - - Map consumerProps = KafkaTestUtils.consumerProps("foobar1", - "false", embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - Consumer consumer1 = cf.createConsumer(); - embeddedKafka.consumeFromEmbeddedTopics(consumer1, "error.foos1.fooz-group", - "error.foos2.fooz-group"); - - ConsumerRecord cr1 = KafkaTestUtils.getSingleRecord(consumer1, - "error.foos1.fooz-group"); - assertThat(cr1.value().equals("hello")).isTrue(); - - ConsumerRecord cr2 = KafkaTestUtils.getSingleRecord(consumer1, - "error.foos2.fooz-group"); - assertThat(cr2.value().equals("hello")).isTrue(); - - // Ensuring that the deserialization was indeed done by the binder - verify(conversionDelegate).deserializeOnInbound(any(Class.class), - any(KStream.class)); - } - - } - - @EnableBinding(KafkaStreamsProcessor.class) - @EnableAutoConfiguration - public static class ProductCountApplication { - - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - return input.filter((key, product) -> product.getId() == 123) - .map((key, value) -> new KeyValue<>(value, value)) - .groupByKey(Grouped.with(new JsonSerde<>(Product.class), - new JsonSerde<>(Product.class))) - .windowedBy(TimeWindows.of(Duration.ofMillis(5000))) - .count(Materialized.as("id-count-store-x")).toStream() - .map((key, value) -> new KeyValue<>(key.key().id, value)); - } - - } - - static class Product { - - Integer id; - - public Integer getId() { - return id; - } - - public void setId(Integer id) { - this.id = id; - } - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderHealthIndicatorTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderHealthIndicatorTests.java index a0694b152..e1a2f1f3f 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderHealthIndicatorTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderHealthIndicatorTests.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -39,12 +40,7 @@ import org.springframework.boot.actuate.health.CompositeHealthContributor; import org.springframework.boot.actuate.health.Health; import org.springframework.boot.actuate.health.Status; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.cloud.stream.annotation.StreamListener; import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsBinderHealthIndicator; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.kafka.config.KafkaStreamsCustomizer; @@ -56,7 +52,6 @@ import org.springframework.kafka.support.SendResult; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; import org.springframework.util.concurrent.ListenableFuture; import org.springframework.util.concurrent.ListenableFutureCallback; @@ -207,6 +202,8 @@ public class KafkaStreamsBinderHealthIndicatorTests { SpringApplication app = new SpringApplication(KStreamApplication.class); app.setWebApplicationType(WebApplicationType.NONE); return app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input", + "--spring.cloud.stream.function.bindings.process-out-0=output", "--spring.cloud.stream.bindings.input.destination=in", "--spring.cloud.stream.bindings.output.destination=out", "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", @@ -225,6 +222,11 @@ public class KafkaStreamsBinderHealthIndicatorTests { SpringApplication app = new SpringApplication(AnotherKStreamApplication.class); app.setWebApplicationType(WebApplicationType.NONE); return app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.function.definition=process;process2", + "--spring.cloud.stream.function.bindings.process-in-0=input", + "--spring.cloud.stream.function.bindings.process-out-0=output", + "--spring.cloud.stream.function.bindings.process2-in-0=input2", + "--spring.cloud.stream.function.bindings.process2-out-0=output2", "--spring.cloud.stream.bindings.input.destination=in", "--spring.cloud.stream.bindings.output.destination=out", "--spring.cloud.stream.bindings.input2.destination=in2", @@ -242,14 +244,12 @@ public class KafkaStreamsBinderHealthIndicatorTests { + embeddedKafka.getBrokersAsString()); } - @EnableBinding(KafkaStreamsProcessor.class) @EnableAutoConfiguration public static class KStreamApplication { - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - return input.filter((key, product) -> { + @Bean + public Function, KStream> process() { + return input -> input.filter((key, product) -> { if (product.getId() != 123) { throw new IllegalArgumentException(); } @@ -259,14 +259,12 @@ public class KafkaStreamsBinderHealthIndicatorTests { } - @EnableBinding({ KafkaStreamsProcessor.class, KafkaStreamsProcessorX.class }) @EnableAutoConfiguration public static class AnotherKStreamApplication { - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - return input.filter((key, product) -> { + @Bean + public Function, KStream> process() { + return input -> input.filter((key, product) -> { if (product.getId() != 123) { throw new IllegalArgumentException(); } @@ -274,10 +272,9 @@ public class KafkaStreamsBinderHealthIndicatorTests { }); } - @StreamListener("input2") - @SendTo("output2") - public KStream process2(KStream input) { - return input.filter((key, product) -> { + @Bean + public Function, KStream> process2() { + return input -> input.filter((key, product) -> { if (product.getId() != 123) { throw new IllegalArgumentException(); } @@ -300,16 +297,6 @@ public class KafkaStreamsBinderHealthIndicatorTests { } - public interface KafkaStreamsProcessorX { - - @Input("input2") - KStream input(); - - @Output("output2") - KStream output(); - - } - public static class Product { Integer id; @@ -323,5 +310,4 @@ public class KafkaStreamsBinderHealthIndicatorTests { } } - } diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderMultipleInputTopicsTest.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderMultipleInputTopicsTest.java index f857c3494..2b841e77f 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderMultipleInputTopicsTest.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderMultipleInputTopicsTest.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.function.Function; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -37,10 +38,6 @@ import org.junit.Test; import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.kafka.core.CleanupConfig; @@ -50,7 +47,6 @@ import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; import static org.assertj.core.api.Assertions.assertThat; @@ -100,6 +96,8 @@ public class KafkaStreamsBinderMultipleInputTopicsTest { ConfigurableApplicationContext context = app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input", + "--spring.cloud.stream.function.bindings.process-out-0=output", "--spring.cloud.stream.bindings.input.destination=words1,words2", "--spring.cloud.stream.bindings.output.destination=counts", "--spring.cloud.stream.bindings.output.contentType=application/json", @@ -146,21 +144,13 @@ public class KafkaStreamsBinderMultipleInputTopicsTest { assertThat(wordCounts.contains("{\"word\":\"foobar2\",\"count\":1}")).isTrue(); } - @EnableBinding(KafkaStreamsProcessor.class) @EnableAutoConfiguration static class WordCountProcessorApplication { - @StreamListener - @SendTo("output") - public KStream process( - @Input("input") KStream input) { + @Bean + public Function, KStream> process() { - input.map((k, v) -> { - System.out.println(k); - System.out.println(v); - return new KeyValue<>(k, v); - }); - return input + return input -> input .flatMapValues( value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .map((key, value) -> new KeyValue<>(value, value)) diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java index bda892780..d95658cee 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java @@ -18,6 +18,7 @@ package org.springframework.cloud.stream.binder.kafka.streams.integration; import java.time.Duration; import java.util.Map; +import java.util.function.Function; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -36,10 +37,8 @@ import org.junit.Test; import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; import org.springframework.kafka.core.DefaultKafkaProducerFactory; import org.springframework.kafka.core.KafkaTemplate; @@ -47,7 +46,6 @@ import org.springframework.kafka.support.serializer.JsonSerde; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; import static org.assertj.core.api.Assertions.assertThat; @@ -89,6 +87,8 @@ public class KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests { app.setWebApplicationType(WebApplicationType.NONE); ConfigurableApplicationContext context = app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input", + "--spring.cloud.stream.function.bindings.process-out-0=output", "--spring.cloud.stream.bindings.input.destination=foos", "--spring.cloud.stream.bindings.output.destination=counts-id", "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", @@ -122,24 +122,19 @@ public class KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests { assertThat(cr.value()).isEqualTo(1L); } - @EnableBinding(KafkaStreamsProcessor.class) @EnableAutoConfiguration public static class ProductCountApplication { - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - return input.filter((key, product) -> product.getId() == 123) + @Bean + public Function, KStream> process() { + return input -> input.filter((key, product) -> product.getId() == 123) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Grouped.with(new JsonSerde<>(Product.class), new JsonSerde<>(Product.class))) .windowedBy(TimeWindows.of(Duration.ofMillis(5000))) .count(Materialized.as("id-count-store-x")).toStream() - .map((key, value) -> { - return new KeyValue<>(key.key().id, value); - }); + .map((key, value) -> new KeyValue<>(key.key().id, value)); } - } public static class Product { diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsNativeEncodingDecodingTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsNativeEncodingDecodingTests.java deleted file mode 100644 index 4bb5abb46..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsNativeEncodingDecodingTests.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.integration; - -import java.time.Duration; -import java.util.Arrays; -import java.util.Map; - -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.kstream.Grouped; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.Materialized; -import org.apache.kafka.streams.kstream.TimeWindows; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.runner.RunWith; - -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.test.EmbeddedKafkaBroker; -import org.springframework.kafka.test.rule.EmbeddedKafkaRule; -import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.util.StopWatch; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; - -/** - * @author Soby Chacko - */ -@RunWith(SpringRunner.class) -@ContextConfiguration -@DirtiesContext -public abstract class KafkaStreamsNativeEncodingDecodingTests { - - @ClassRule - public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, - "decode-counts", "decode-counts-1"); - - private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule - .getEmbeddedKafka(); - - @SpyBean - org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsMessageConversionDelegate conversionDelegate; - - private static Consumer consumer; - - @BeforeClass - public static void setUp() { - System.setProperty("spring.cloud.stream.kafka.streams.binder.brokers", - embeddedKafka.getBrokersAsString()); - System.setProperty("server.port", "0"); - System.setProperty("spring.jmx.enabled", "false"); - - Map consumerProps = KafkaTestUtils.consumerProps("group", "false", - embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - consumer = cf.createConsumer(); - embeddedKafka.consumeFromEmbeddedTopics(consumer, "decode-counts", "decode-counts-1"); - } - - @AfterClass - public static void tearDown() { - consumer.close(); - System.clearProperty("spring.cloud.stream.kafka.streams.binder.brokers"); - System.clearProperty("server.port"); - System.clearProperty("spring.jmx.enabled"); - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.destination=decode-words-1", - "spring.cloud.stream.bindings.output.destination=decode-counts-1", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId" - + "=NativeEncodingDecodingEnabledTests-abc" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - public static class NativeEncodingDecodingEnabledTests - extends KafkaStreamsNativeEncodingDecodingTests { - - @Test - public void test() throws Exception { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("decode-words-1"); - template.sendDefault("foobar"); - ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, - "decode-counts-1"); - assertThat(cr.value().equals("Count for foobar : 1")).isTrue(); - - verify(conversionDelegate, never()).serializeOnOutbound(any(KStream.class)); - verify(conversionDelegate, never()).deserializeOnInbound(any(Class.class), - any(KStream.class)); - } - - } - - @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { - "spring.cloud.stream.bindings.input.destination=decode-words", - "spring.cloud.stream.bindings.output.destination=decode-counts", - "spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", - "spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", - "spring.cloud.stream.kafka.streams.bindings.input3.consumer.applicationId" - + "=hello-NativeEncodingDecodingEnabledTests-xyz" }) - public static class NativeEncodingDecodingDisabledTests - extends KafkaStreamsNativeEncodingDecodingTests { - - @Test - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("decode-words"); - template.sendDefault("foobar"); - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - System.out.println("Starting: "); - ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, - "decode-counts"); - stopWatch.stop(); - System.out.println("Total time: " + stopWatch.getTotalTimeSeconds()); - assertThat(cr.value().equals("Count for foobar : 1")).isTrue(); - - verify(conversionDelegate).serializeOnOutbound(any(KStream.class)); - verify(conversionDelegate).deserializeOnInbound(any(Class.class), - any(KStream.class)); - } - - } - - @EnableBinding(KafkaStreamsProcessor.class) - @EnableAutoConfiguration - public static class WordCountProcessorApplication { - - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - - return input - .flatMapValues( - value -> Arrays.asList(value.toLowerCase().split("\\W+"))) - .map((key, value) -> new KeyValue<>(value, value)) - .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) - .windowedBy(TimeWindows.of(Duration.ofSeconds(5))).count(Materialized.as("foo-WordCounts-x")) - .toStream().map((key, value) -> new KeyValue<>(null, - "Count for " + key.key() + " : " + value)); - } - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsStateStoreIntegrationTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsStateStoreIntegrationTests.java index 2cc15053e..22f0bf06d 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsStateStoreIntegrationTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsStateStoreIntegrationTests.java @@ -18,6 +18,8 @@ package org.springframework.cloud.stream.binder.kafka.streams.integration; import java.time.Duration; import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Consumer; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.kstream.KStream; @@ -32,11 +34,6 @@ import org.junit.Test; import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsStateStore; -import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsStateStoreProperties; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.kafka.core.DefaultKafkaProducerFactory; @@ -67,6 +64,7 @@ public class KafkaStreamsStateStoreIntegrationTests { app.setWebApplicationType(WebApplicationType.NONE); ConfigurableApplicationContext context = app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input", "--spring.cloud.stream.bindings.input.destination=foobar", "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" @@ -89,41 +87,14 @@ public class KafkaStreamsStateStoreIntegrationTests { } } - @Test - public void testKstreamStateStoreBuilderBeansDefinedInApplication() throws Exception { - SpringApplication app = new SpringApplication(StateStoreBeanApplication.class); - app.setWebApplicationType(WebApplicationType.NONE); - ConfigurableApplicationContext context = app.run("--server.port=0", - "--spring.jmx.enabled=false", - "--spring.cloud.stream.bindings.input3.destination=foobar", - "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", - "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" - + "=org.apache.kafka.common.serialization.Serdes$StringSerde", - "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" - + "=org.apache.kafka.common.serialization.Serdes$StringSerde", - "--spring.cloud.stream.kafka.streams.bindings.input3.consumer.applicationId" - + "=KafkaStreamsStateStoreIntegrationTests-xyzabc-123", - "--spring.cloud.stream.kafka.streams.binder.brokers=" - + embeddedKafka.getBrokersAsString()); - try { - Thread.sleep(2000); - receiveAndValidateFoo(context, StateStoreBeanApplication.class); - } - catch (Exception e) { - throw e; - } - finally { - context.close(); - } - } - - @Test public void testSameStateStoreIsCreatedOnlyOnceWhenMultipleInputBindingsArePresent() throws Exception { SpringApplication app = new SpringApplication(ProductCountApplicationWithMultipleInputBindings.class); app.setWebApplicationType(WebApplicationType.NONE); ConfigurableApplicationContext context = app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input1", + "--spring.cloud.stream.function.bindings.process-in-1=input2", "--spring.cloud.stream.bindings.input1.destination=foobar", "--spring.cloud.stream.bindings.input2.destination=hello-foobar", "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", @@ -171,22 +142,12 @@ public class KafkaStreamsStateStoreIntegrationTests { assertThat(state.persistent()).isTrue(); assertThat(productCount.processed).isTrue(); } - else if (clazz.isAssignableFrom(StateStoreBeanApplication.class)) { - StateStoreBeanApplication productCount = context - .getBean(StateStoreBeanApplication.class); - WindowStore state = productCount.state; - assertThat(state != null).isTrue(); - assertThat(state.name()).isEqualTo("mystate"); - assertThat(state.persistent()).isTrue(); - assertThat(productCount.processed).isTrue(); - } else { - fail("Expected assertiond did not happen"); + fail("Expected assertions did not happen"); } } - @EnableBinding(KafkaStreamsProcessorX.class) @EnableAutoConfiguration public static class ProductCountApplication { @@ -194,46 +155,10 @@ public class KafkaStreamsStateStoreIntegrationTests { boolean processed; - @StreamListener("input") - @KafkaStreamsStateStore(name = "mystate", type = KafkaStreamsStateStoreProperties.StoreType.WINDOW, lengthMs = 300000, retentionMs = 300000) - @SuppressWarnings({ "deprecation", "unchecked" }) - public void process(KStream input) { + @Bean + public Consumer> process() { - input.process(() -> new Processor() { - - @Override - public void init(ProcessorContext processorContext) { - state = (WindowStore) processorContext.getStateStore("mystate"); - } - - @Override - public void process(Object s, Product product) { - processed = true; - } - - @Override - public void close() { - if (state != null) { - state.close(); - } - } - }, "mystate"); - } - } - - @EnableBinding(KafkaStreamsProcessorZ.class) - @EnableAutoConfiguration - public static class StateStoreBeanApplication { - - WindowStore state; - - boolean processed; - - @StreamListener("input3") - @SuppressWarnings({"unchecked" }) - public void process(KStream input) { - - input.process(() -> new Processor() { + return input -> input.process(() -> new Processor() { @Override public void init(ProcessorContext processorContext) { @@ -263,8 +188,6 @@ public class KafkaStreamsStateStoreIntegrationTests { } } - - @EnableBinding(KafkaStreamsProcessorY.class) @EnableAutoConfiguration public static class ProductCountApplicationWithMultipleInputBindings { @@ -272,33 +195,41 @@ public class KafkaStreamsStateStoreIntegrationTests { boolean processed; - @StreamListener - @KafkaStreamsStateStore(name = "mystate", type = KafkaStreamsStateStoreProperties.StoreType.WINDOW, lengthMs = 300000, retentionMs = 300000) - @SuppressWarnings({ "deprecation", "unchecked" }) - public void process(@Input("input1")KStream input, @Input("input2")KStream input2) { + @Bean + public BiConsumer, KStream> process() { - input.process(() -> new Processor() { + return (input, input2) -> { - @Override - public void init(ProcessorContext processorContext) { - state = (WindowStore) processorContext.getStateStore("mystate"); - } + input.process(() -> new Processor() { - @Override - public void process(Object s, Product product) { - processed = true; - } - - @Override - public void close() { - if (state != null) { - state.close(); + @Override + public void init(ProcessorContext processorContext) { + state = (WindowStore) processorContext.getStateStore("mystate"); } - } - }, "mystate"); - //simple use of input2, we are not using input2 for anything other than triggering some test behavior. - input2.foreach((key, value) -> { }); + @Override + public void process(Object s, Product product) { + processed = true; + } + + @Override + public void close() { + if (state != null) { + state.close(); + } + } + }, "mystate"); + //simple use of input2, we are not using input2 for anything other than triggering some test behavior. + input2.foreach((key, value) -> { }); + }; + } + + @Bean + public StoreBuilder mystore() { + return Stores.windowStoreBuilder( + Stores.persistentWindowStore("mystate", + Duration.ofMillis(3), Duration.ofMillis(3), false), Serdes.String(), + Serdes.String()); } } @@ -315,25 +246,4 @@ public class KafkaStreamsStateStoreIntegrationTests { } } - - interface KafkaStreamsProcessorX { - - @Input("input") - KStream input(); - } - - interface KafkaStreamsProcessorY { - - @Input("input1") - KStream input1(); - - @Input("input2") - KStream input2(); - } - - interface KafkaStreamsProcessorZ { - - @Input("input3") - KStream input3(); - } } diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkastreamsBinderPojoInputStringOutputIntegrationTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkastreamsBinderPojoInputStringOutputIntegrationTests.java index 2ff80f252..130db7be4 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkastreamsBinderPojoInputStringOutputIntegrationTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkastreamsBinderPojoInputStringOutputIntegrationTests.java @@ -18,6 +18,7 @@ package org.springframework.cloud.stream.binder.kafka.streams.integration; import java.time.Duration; import java.util.Map; +import java.util.function.Function; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -35,10 +36,8 @@ import org.junit.Test; import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; import org.springframework.integration.test.util.TestUtils; import org.springframework.kafka.config.StreamsBuilderFactoryBean; import org.springframework.kafka.core.CleanupConfig; @@ -49,7 +48,6 @@ import org.springframework.kafka.support.serializer.JsonSerde; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; import static org.assertj.core.api.Assertions.assertThat; @@ -91,6 +89,8 @@ public class KafkastreamsBinderPojoInputStringOutputIntegrationTests { app.setWebApplicationType(WebApplicationType.NONE); ConfigurableApplicationContext context = app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input", + "--spring.cloud.stream.function.bindings.process-out-0=output", "--spring.cloud.stream.bindings.input.destination=foos", "--spring.cloud.stream.bindings.output.destination=counts-id", "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", @@ -105,7 +105,7 @@ public class KafkastreamsBinderPojoInputStringOutputIntegrationTests { receiveAndValidateFoo(); // Assertions on StreamBuilderFactoryBean StreamsBuilderFactoryBean streamsBuilderFactoryBean = context - .getBean("&stream-builder-ProductCountApplication-process", StreamsBuilderFactoryBean.class); + .getBean("&stream-builder-process", StreamsBuilderFactoryBean.class); CleanupConfig cleanup = TestUtils.getPropertyValue(streamsBuilderFactoryBean, "cleanupConfig", CleanupConfig.class); assertThat(cleanup.cleanupOnStart()).isFalse(); @@ -128,15 +128,12 @@ public class KafkastreamsBinderPojoInputStringOutputIntegrationTests { assertThat(cr.value().contains("Count for product with ID 123: 1")).isTrue(); } - @EnableBinding(KafkaStreamsProcessor.class) @EnableAutoConfiguration public static class ProductCountApplication { - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - - return input.filter((key, product) -> product.getId() == 123) + @Bean + public Function, KStream> process() { + return input -> input.filter((key, product) -> product.getId() == 123) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Grouped.with(new JsonSerde<>(Product.class), new JsonSerde<>(Product.class))) diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/MultiProcessorsWithSameNameAndBindingTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/MultiProcessorsWithSameNameAndBindingTests.java deleted file mode 100644 index 699cee4a7..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/MultiProcessorsWithSameNameAndBindingTests.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2019-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.integration; - -import org.apache.kafka.streams.kstream.KStream; -import org.junit.ClassRule; -import org.junit.Test; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.WebApplicationType; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.kafka.config.StreamsBuilderFactoryBean; -import org.springframework.kafka.test.EmbeddedKafkaBroker; -import org.springframework.kafka.test.rule.EmbeddedKafkaRule; -import org.springframework.stereotype.Component; - -import static org.assertj.core.api.Assertions.assertThat; - -public class MultiProcessorsWithSameNameAndBindingTests { - - @ClassRule - public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, - "counts"); - - private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule - .getEmbeddedKafka(); - - @Test - public void testBinderStartsSuccessfullyWhenTwoProcessorsWithSameNamesAndBindingsPresent() { - SpringApplication app = new SpringApplication( - MultiProcessorsWithSameNameAndBindingTests.WordCountProcessorApplication.class); - app.setWebApplicationType(WebApplicationType.NONE); - - try (ConfigurableApplicationContext context = app.run("--server.port=0", - "--spring.jmx.enabled=false", - "--spring.cloud.stream.bindings.input.destination=words", - "--spring.cloud.stream.bindings.input-1.destination=words", - "--spring.cloud.stream.bindings.output.destination=counts", - "--spring.cloud.stream.bindings.output.contentType=application/json", - "--spring.cloud.stream.kafka.streams.binder.brokers=" - + embeddedKafka.getBrokersAsString())) { - StreamsBuilderFactoryBean streamsBuilderFactoryBean1 = context - .getBean("&stream-builder-Foo-process", StreamsBuilderFactoryBean.class); - assertThat(streamsBuilderFactoryBean1).isNotNull(); - StreamsBuilderFactoryBean streamsBuilderFactoryBean2 = context - .getBean("&stream-builder-Bar-process", StreamsBuilderFactoryBean.class); - assertThat(streamsBuilderFactoryBean2).isNotNull(); - } - } - - @EnableBinding(KafkaStreamsProcessorX.class) - @EnableAutoConfiguration - static class WordCountProcessorApplication { - - @Component - static class Foo { - @StreamListener - public void process(@Input("input-1") KStream input) { - } - } - - //Second class with a stub processor that has the same name as above ("process") - @Component - static class Bar { - @StreamListener - public void process(@Input("input-1") KStream input) { - } - } - } - - interface KafkaStreamsProcessorX { - - @Input("input-1") - KStream input1(); - - } -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/PerRecordAvroContentTypeTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/PerRecordAvroContentTypeTests.java deleted file mode 100644 index 59eee2591..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/PerRecordAvroContentTypeTests.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright 2017-2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.integration; - -import java.io.IOException; -import java.util.Map; -import java.util.Random; -import java.util.UUID; - -import com.example.Sensor; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.ByteArrayDeserializer; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.kstream.KStream; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.WebApplicationType; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.function.context.converter.avro.AvroSchemaMessageConverter; -import org.springframework.cloud.function.context.converter.avro.AvroSchemaServiceManagerImpl; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; -import org.springframework.cloud.stream.binder.kafka.streams.integration.utils.TestAvroSerializer; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.annotation.Bean; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.test.EmbeddedKafkaBroker; -import org.springframework.kafka.test.rule.EmbeddedKafkaRule; -import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.Message; -import org.springframework.messaging.converter.MessageConverter; -import org.springframework.messaging.handler.annotation.SendTo; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.util.MimeTypeUtils; - -import static org.assertj.core.api.Assertions.assertThat; - - -/** - * @author Soby Chacko - */ -public class PerRecordAvroContentTypeTests { - - @ClassRule - public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, - "received-sensors"); - - private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule - .getEmbeddedKafka(); - - private static Consumer consumer; - - @BeforeClass - public static void setUp() throws Exception { - Map consumerProps = KafkaTestUtils.consumerProps("avro-ct-test", - "false", embeddedKafka); - - // Receive the data as byte[] - consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - ByteArrayDeserializer.class); - - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - consumer = cf.createConsumer(); - embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "received-sensors"); - } - - @AfterClass - public static void tearDown() { - consumer.close(); - } - - @Test - public void testPerRecordAvroConentTypeAndVerifySerialization() throws Exception { - SpringApplication app = new SpringApplication(SensorCountAvroApplication.class); - app.setWebApplicationType(WebApplicationType.NONE); - - try (ConfigurableApplicationContext ignored = app.run("--server.port=0", - "--spring.jmx.enabled=false", - "--spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", - "--spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", - "--spring.cloud.stream.bindings.input.destination=sensors", - "--spring.cloud.stream.bindings.output.destination=received-sensors", - "--spring.cloud.stream.bindings.output.contentType=application/avro", - "--spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=per-record-avro-contentType-test", - "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", - "--spring.cloud.stream.kafka.streams.binder.brokers=" - + embeddedKafka.getBrokersAsString())) { - - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - // Use a custom avro test serializer - senderProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - TestAvroSerializer.class); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - try { - KafkaTemplate template = new KafkaTemplate<>(pf, true); - - Random random = new Random(); - Sensor sensor = new Sensor(); - sensor.setId(UUID.randomUUID().toString() + "-v1"); - sensor.setAcceleration(random.nextFloat() * 10); - sensor.setVelocity(random.nextFloat() * 100); - sensor.setTemperature(random.nextFloat() * 50); - // Send with avro content type set. - Message message = MessageBuilder.withPayload(sensor) - .setHeader("contentType", "application/avro").build(); - template.setDefaultTopic("sensors"); - template.send(message); - - // Serialized byte[] ^^ is received by the binding process and deserialzed - // it using avro converter. - // Then finally, the data will be output to a return topic as byte[] - // (using the same avro converter). - - // Receive the byte[] from return topic - ConsumerRecord cr = KafkaTestUtils - .getSingleRecord(consumer, "received-sensors"); - final byte[] value = cr.value(); - - // Convert the byte[] received back to avro object and verify that it is - // the same as the one we sent ^^. - AvroSchemaMessageConverter avroSchemaMessageConverter = new AvroSchemaMessageConverter(new AvroSchemaServiceManagerImpl()); - - Message receivedMessage = MessageBuilder.withPayload(value) - .setHeader("contentType", - MimeTypeUtils.parseMimeType("application/avro")) - .build(); - Sensor messageConverted = (Sensor) avroSchemaMessageConverter - .fromMessage(receivedMessage, Sensor.class); - assertThat(messageConverted).isEqualTo(sensor); - } - finally { - pf.destroy(); - } - } - } - - @EnableBinding(KafkaStreamsProcessor.class) - @EnableAutoConfiguration - static class SensorCountAvroApplication { - - @StreamListener - @SendTo("output") - public KStream process(@Input("input") KStream input) { - // return the same Sensor object unchanged so that we can do test - // verifications - return input.map(KeyValue::new); - } - - @Bean - public MessageConverter sensorMessageConverter() throws IOException { - return new AvroSchemaMessageConverter(new AvroSchemaServiceManagerImpl()); - } - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/utils/TestAvroSerializer.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/utils/TestAvroSerializer.java deleted file mode 100644 index 761636ea9..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/utils/TestAvroSerializer.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.integration.utils; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.kafka.common.serialization.Serializer; - -import org.springframework.cloud.function.context.converter.avro.AvroSchemaMessageConverter; -import org.springframework.cloud.function.context.converter.avro.AvroSchemaServiceManagerImpl; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.support.MessageBuilder; - -/** - * Custom avro serializer intended to be used for testing only. - * - * @param Target type to serialize - * @author Soby Chacko - */ -public class TestAvroSerializer implements Serializer { - - public TestAvroSerializer() { - } - - @Override - public void configure(Map configs, boolean isKey) { - - } - - @Override - public byte[] serialize(String topic, S data) { - AvroSchemaMessageConverter avroSchemaMessageConverter = new AvroSchemaMessageConverter(new AvroSchemaServiceManagerImpl()); - Message message = MessageBuilder.withPayload(data).build(); - Map headers = new HashMap<>(message.getHeaders()); - headers.put(MessageHeaders.CONTENT_TYPE, "application/avro"); - MessageHeaders messageHeaders = new MessageHeaders(headers); - final Object payload = avroSchemaMessageConverter - .toMessage(message.getPayload(), messageHeaders).getPayload(); - return (byte[]) payload; - } - - @Override - public void close() { - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/resources/org/springframework/cloud/stream/binder/kstream/integTest-1.properties b/spring-cloud-stream-binder-kafka-streams/src/test/resources/org/springframework/cloud/stream/binder/kstream/integTest-1.properties deleted file mode 100644 index 6d983a0b1..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/resources/org/springframework/cloud/stream/binder/kstream/integTest-1.properties +++ /dev/null @@ -1,6 +0,0 @@ -spring.cloud.stream.bindings.input.destination=DeserializationErrorHandlerByKafkaTests-In -spring.cloud.stream.bindings.output.destination=DeserializationErrorHandlerByKafkaTests-Out -spring.cloud.stream.bindings.output.contentType=application/json -spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000 -spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde=org.apache.kafka.common.serialization.Serdes$StringSerde -spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde=org.apache.kafka.common.serialization.Serdes$StringSerde From ed98f1129da9e9fa8aa6208b3a59f27f67834609 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Tue, 9 Nov 2021 13:56:10 -0500 Subject: [PATCH 02/19] Kafka Streams binder deprecated component removals --- .../pom.xml | 29 - ...KStreamStreamListenerParameterAdapter.java | 66 --- .../KStreamStreamListenerResultAdapter.java | 58 -- ...StreamsBinderSupportAutoConfiguration.java | 49 -- ...StreamListenerSetupMethodOrchestrator.java | 521 ------------------ .../annotations/KafkaStreamsProcessor.java | 90 --- .../annotations/KafkaStreamsStateStore.java | 115 ---- .../KafkaStreamsStateStoreProperties.java | 161 ------ .../serde/CompositeNonNativeSerde.java | 37 -- .../serde/MessageConverterDelegateSerde.java | 228 -------- .../src/test/resources/avro/sensor.avsc | 11 - 11 files changed, 1365 deletions(-) delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerParameterAdapter.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerResultAdapter.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsStreamListenerSetupMethodOrchestrator.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsProcessor.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsStateStore.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsStateStoreProperties.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CompositeNonNativeSerde.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/MessageConverterDelegateSerde.java delete mode 100644 spring-cloud-stream-binder-kafka-streams/src/test/resources/avro/sensor.avsc diff --git a/spring-cloud-stream-binder-kafka-streams/pom.xml b/spring-cloud-stream-binder-kafka-streams/pom.xml index feb13e631..f68b11e69 100644 --- a/spring-cloud-stream-binder-kafka-streams/pom.xml +++ b/spring-cloud-stream-binder-kafka-streams/pom.xml @@ -73,35 +73,6 @@ kafka_2.13 test - - - org.apache.avro - avro - ${avro.version} - provided - - - - - org.apache.avro - avro-maven-plugin - ${avro.version} - - - generate-test-sources - - schema - - - ${project.basedir}/target/generated-test-sources - ${project.basedir}/target/generated-test-sources - ${project.basedir}/src/test/resources/avro - - - - - - diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerParameterAdapter.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerParameterAdapter.java deleted file mode 100644 index d2645c957..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerParameterAdapter.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2017-2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams; - -import org.apache.kafka.streams.kstream.KStream; - -import org.springframework.cloud.stream.binding.StreamListenerParameterAdapter; -import org.springframework.core.MethodParameter; -import org.springframework.core.ResolvableType; - -/** - * {@link StreamListenerParameterAdapter} for KStream. - * - * @author Marius Bogoevici - * @author Soby Chacko - */ -class KStreamStreamListenerParameterAdapter - implements StreamListenerParameterAdapter, KStream> { - - private final KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate; - - private final KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue; - - KStreamStreamListenerParameterAdapter( - KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate, - KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue) { - this.kafkaStreamsMessageConversionDelegate = kafkaStreamsMessageConversionDelegate; - this.KafkaStreamsBindingInformationCatalogue = KafkaStreamsBindingInformationCatalogue; - } - - @Override - public boolean supports(Class bindingTargetType, MethodParameter methodParameter) { - return KafkaStreamsBinderUtils.supportsKStream(methodParameter, bindingTargetType); - } - - @Override - @SuppressWarnings("unchecked") - public KStream adapt(KStream bindingTarget, MethodParameter parameter) { - ResolvableType resolvableType = ResolvableType.forMethodParameter(parameter); - final Class valueClass = (resolvableType.getGeneric(1).getRawClass() != null) - ? (resolvableType.getGeneric(1).getRawClass()) : Object.class; - if (this.KafkaStreamsBindingInformationCatalogue - .isUseNativeDecoding(bindingTarget)) { - return bindingTarget; - } - else { - return this.kafkaStreamsMessageConversionDelegate - .deserializeOnInbound(valueClass, bindingTarget); - } - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerResultAdapter.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerResultAdapter.java deleted file mode 100644 index 6ffce5732..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerResultAdapter.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2017-2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams; - -import java.io.Closeable; -import java.io.IOException; - -import org.apache.kafka.streams.kstream.KStream; - -import org.springframework.cloud.stream.binding.StreamListenerResultAdapter; - -/** - * {@link StreamListenerResultAdapter} for KStream. - * - * @author Marius Bogoevici - * @author Soby Chacko - */ -class KStreamStreamListenerResultAdapter implements - StreamListenerResultAdapter { - - @Override - public boolean supports(Class resultType, Class boundElement) { - return KStream.class.isAssignableFrom(resultType) - && KStream.class.isAssignableFrom(boundElement); - } - - @Override - @SuppressWarnings("unchecked") - public Closeable adapt(KStream streamListenerResult, - KStreamBoundElementFactory.KStreamWrapper boundElement) { - boundElement.wrap(streamListenerResult); - return new NoOpCloseable(); - } - - private static final class NoOpCloseable implements Closeable { - - @Override - public void close() throws IOException { - - } - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderSupportAutoConfiguration.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderSupportAutoConfiguration.java index 17c4d8723..e522988b8 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderSupportAutoConfiguration.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderSupportAutoConfiguration.java @@ -17,7 +17,6 @@ package org.springframework.cloud.stream.binder.kafka.streams; import java.lang.reflect.Constructor; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,10 +49,7 @@ import org.springframework.cloud.stream.binder.BinderConfiguration; import org.springframework.cloud.stream.binder.kafka.streams.function.FunctionDetectorCondition; import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; -import org.springframework.cloud.stream.binder.kafka.streams.serde.CompositeNonNativeSerde; -import org.springframework.cloud.stream.binder.kafka.streams.serde.MessageConverterDelegateSerde; import org.springframework.cloud.stream.binding.BindingService; -import org.springframework.cloud.stream.binding.StreamListenerResultAdapter; import org.springframework.cloud.stream.config.BinderProperties; import org.springframework.cloud.stream.config.BindingServiceConfiguration; import org.springframework.cloud.stream.config.BindingServiceProperties; @@ -296,37 +292,6 @@ public class KafkaStreamsBinderSupportAutoConfiguration { } } - @Bean - public KStreamStreamListenerResultAdapter kstreamStreamListenerResultAdapter() { - return new KStreamStreamListenerResultAdapter(); - } - - @Bean - public KStreamStreamListenerParameterAdapter kstreamStreamListenerParameterAdapter( - KafkaStreamsMessageConversionDelegate kstreamBoundMessageConversionDelegate, - KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue) { - return new KStreamStreamListenerParameterAdapter( - kstreamBoundMessageConversionDelegate, - KafkaStreamsBindingInformationCatalogue); - } - - @Bean - public KafkaStreamsStreamListenerSetupMethodOrchestrator kafkaStreamsStreamListenerSetupMethodOrchestrator( - BindingServiceProperties bindingServiceProperties, - KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties, - KeyValueSerdeResolver keyValueSerdeResolver, - KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, - KStreamStreamListenerParameterAdapter kafkaStreamListenerParameterAdapter, - Collection streamListenerResultAdapters, - ObjectProvider cleanupConfig, - ObjectProvider customizerProvider, ConfigurableEnvironment environment) { - return new KafkaStreamsStreamListenerSetupMethodOrchestrator( - bindingServiceProperties, kafkaStreamsExtendedBindingProperties, - keyValueSerdeResolver, kafkaStreamsBindingInformationCatalogue, - kafkaStreamListenerParameterAdapter, streamListenerResultAdapters, - cleanupConfig.getIfUnique(), customizerProvider.getIfUnique(), environment); - } - @Bean public KafkaStreamsMessageConversionDelegate messageConversionDelegate( @Qualifier(IntegrationContextUtils.ARGUMENT_RESOLVER_MESSAGE_CONVERTER_BEAN_NAME) @@ -338,20 +303,6 @@ public class KafkaStreamsBinderSupportAutoConfiguration { KafkaStreamsBindingInformationCatalogue, binderConfigurationProperties); } - @Bean - public MessageConverterDelegateSerde messageConverterDelegateSerde( - @Qualifier(IntegrationContextUtils.ARGUMENT_RESOLVER_MESSAGE_CONVERTER_BEAN_NAME) - CompositeMessageConverter compositeMessageConverterFactory) { - return new MessageConverterDelegateSerde(compositeMessageConverterFactory); - } - - @Bean - public CompositeNonNativeSerde compositeNonNativeSerde( - @Qualifier(IntegrationContextUtils.ARGUMENT_RESOLVER_MESSAGE_CONVERTER_BEAN_NAME) - CompositeMessageConverter compositeMessageConverterFactory) { - return new CompositeNonNativeSerde(compositeMessageConverterFactory); - } - @Bean public KStreamBoundElementFactory kStreamBoundElementFactory( BindingServiceProperties bindingServiceProperties, diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsStreamListenerSetupMethodOrchestrator.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsStreamListenerSetupMethodOrchestrator.java deleted file mode 100644 index 6e315faf1..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsStreamListenerSetupMethodOrchestrator.java +++ /dev/null @@ -1,521 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams; - -import java.lang.reflect.Method; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.streams.StreamsBuilder; -import org.apache.kafka.streams.StreamsConfig; -import org.apache.kafka.streams.Topology; -import org.apache.kafka.streams.kstream.GlobalKTable; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.KTable; -import org.apache.kafka.streams.state.StoreBuilder; -import org.apache.kafka.streams.state.Stores; - -import org.springframework.beans.factory.BeanInitializationException; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsStateStore; -import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; -import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; -import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsStateStoreProperties; -import org.springframework.cloud.stream.binding.StreamListenerErrorMessages; -import org.springframework.cloud.stream.binding.StreamListenerParameterAdapter; -import org.springframework.cloud.stream.binding.StreamListenerResultAdapter; -import org.springframework.cloud.stream.binding.StreamListenerSetupMethodOrchestrator; -import org.springframework.cloud.stream.config.BindingProperties; -import org.springframework.cloud.stream.config.BindingServiceProperties; -import org.springframework.context.ApplicationContext; -import org.springframework.core.MethodParameter; -import org.springframework.core.ResolvableType; -import org.springframework.core.annotation.AnnotationUtils; -import org.springframework.core.env.ConfigurableEnvironment; -import org.springframework.kafka.config.StreamsBuilderFactoryBean; -import org.springframework.kafka.config.StreamsBuilderFactoryBeanConfigurer; -import org.springframework.kafka.core.CleanupConfig; -import org.springframework.messaging.handler.annotation.SendTo; -import org.springframework.util.Assert; -import org.springframework.util.ObjectUtils; -import org.springframework.util.ReflectionUtils; -import org.springframework.util.StringUtils; - -/** - * Kafka Streams specific implementation for {@link StreamListenerSetupMethodOrchestrator} - * that overrides the default mechanisms for invoking StreamListener adapters. - *

- * The orchestration primarily focus on the following areas: - *

- * 1. Allow multiple KStream output bindings (KStream branching) by allowing more than one - * output values on {@link SendTo} 2. Allow multiple inbound bindings for multiple KStream - * and or KTable/GlobalKTable types. 3. Each StreamListener method that it orchestrates - * gets its own {@link StreamsBuilderFactoryBean} and {@link StreamsConfig} - * - * @author Soby Chacko - * @author Lei Chen - * @author Gary Russell - */ -class KafkaStreamsStreamListenerSetupMethodOrchestrator extends AbstractKafkaStreamsBinderProcessor - implements StreamListenerSetupMethodOrchestrator { - - private static final Log LOG = LogFactory - .getLog(KafkaStreamsStreamListenerSetupMethodOrchestrator.class); - - private final StreamListenerParameterAdapter streamListenerParameterAdapter; - - private final Collection streamListenerResultAdapters; - - private final BindingServiceProperties bindingServiceProperties; - - private final KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties; - - private final KeyValueSerdeResolver keyValueSerdeResolver; - - private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; - - private final Map> registeredStoresPerMethod = new HashMap<>(); - - private final Map methodStreamsBuilderFactoryBeanMap = new HashMap<>(); - - StreamsBuilderFactoryBeanConfigurer customizer; - - private final ConfigurableEnvironment environment; - - KafkaStreamsStreamListenerSetupMethodOrchestrator( - BindingServiceProperties bindingServiceProperties, - KafkaStreamsExtendedBindingProperties extendedBindingProperties, - KeyValueSerdeResolver keyValueSerdeResolver, - KafkaStreamsBindingInformationCatalogue bindingInformationCatalogue, - StreamListenerParameterAdapter streamListenerParameterAdapter, - Collection listenerResultAdapters, - CleanupConfig cleanupConfig, - StreamsBuilderFactoryBeanConfigurer customizer, - ConfigurableEnvironment environment) { - super(bindingServiceProperties, bindingInformationCatalogue, extendedBindingProperties, keyValueSerdeResolver, cleanupConfig); - this.bindingServiceProperties = bindingServiceProperties; - this.kafkaStreamsExtendedBindingProperties = extendedBindingProperties; - this.keyValueSerdeResolver = keyValueSerdeResolver; - this.kafkaStreamsBindingInformationCatalogue = bindingInformationCatalogue; - this.streamListenerParameterAdapter = streamListenerParameterAdapter; - this.streamListenerResultAdapters = listenerResultAdapters; - this.customizer = customizer; - this.environment = environment; - } - - @Override - public boolean supports(Method method) { - return methodParameterSupports(method) && (methodReturnTypeSuppports(method) - || Void.TYPE.equals(method.getReturnType())); - } - - private boolean methodReturnTypeSuppports(Method method) { - Class returnType = method.getReturnType(); - if (returnType.equals(KStream.class) || (returnType.isArray() - && returnType.getComponentType().equals(KStream.class))) { - return true; - } - return false; - } - - private boolean methodParameterSupports(Method method) { - boolean supports = false; - for (int i = 0; i < method.getParameterCount(); i++) { - MethodParameter methodParameter = MethodParameter.forExecutable(method, i); - Class parameterType = methodParameter.getParameterType(); - if (parameterType.equals(KStream.class) || parameterType.equals(KTable.class) - || parameterType.equals(GlobalKTable.class)) { - supports = true; - } - } - return supports; - } - - @Override - @SuppressWarnings({"rawtypes", "unchecked"}) - public void orchestrateStreamListenerSetupMethod(StreamListener streamListener, - Method method, Object bean) { - String[] methodAnnotatedOutboundNames = getOutboundBindingTargetNames(method); - validateStreamListenerMethod(streamListener, method, - methodAnnotatedOutboundNames); - String methodAnnotatedInboundName = streamListener.value(); - Object[] adaptedInboundArguments = adaptAndRetrieveInboundArguments(method, - methodAnnotatedInboundName, this.applicationContext, - this.streamListenerParameterAdapter); - try { - ReflectionUtils.makeAccessible(method); - if (Void.TYPE.equals(method.getReturnType())) { - method.invoke(bean, adaptedInboundArguments); - } - else { - Object result = method.invoke(bean, adaptedInboundArguments); - - if (methodAnnotatedOutboundNames != null && methodAnnotatedOutboundNames.length > 0) { - if (result.getClass().isArray()) { - Assert.isTrue( - methodAnnotatedOutboundNames.length == ((Object[]) result).length, - "Result does not match with the number of declared outbounds"); - } - else { - Assert.isTrue(methodAnnotatedOutboundNames.length == 1, - "Result does not match with the number of declared outbounds"); - } - } - - if (methodAnnotatedOutboundNames != null && methodAnnotatedOutboundNames.length > 0) { - methodAnnotatedInboundName = populateInboundIfMissing(method, methodAnnotatedInboundName); - final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsBindingInformationCatalogue - .getStreamsBuilderFactoryBeanPerBinding().get(methodAnnotatedInboundName); - - if (result.getClass().isArray()) { - Object[] outboundKStreams = (Object[]) result; - int i = 0; - for (Object outboundKStream : outboundKStreams) { - final String methodAnnotatedOutboundName = methodAnnotatedOutboundNames[i++]; - - this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding( - methodAnnotatedOutboundName, streamsBuilderFactoryBean); - - Object targetBean = this.applicationContext - .getBean(methodAnnotatedOutboundName); - kafkaStreamsBindingInformationCatalogue.addOutboundKStreamResolvable(targetBean, ResolvableType.forMethodReturnType(method)); - adaptStreamListenerResult(outboundKStream, targetBean); - } - } - else { - this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding( - methodAnnotatedOutboundNames[0], streamsBuilderFactoryBean); - - Object targetBean = this.applicationContext - .getBean(methodAnnotatedOutboundNames[0]); - kafkaStreamsBindingInformationCatalogue.addOutboundKStreamResolvable(targetBean, ResolvableType.forMethodReturnType(method)); - adaptStreamListenerResult(result, targetBean); - } - } - } - } - catch (Exception ex) { - throw new BeanInitializationException( - "Cannot setup StreamListener for " + method, ex); - } - } - - private String populateInboundIfMissing(Method method, String methodAnnotatedInboundName) { - if (!StringUtils.hasText(methodAnnotatedInboundName)) { - Object[] arguments = new Object[method.getParameterTypes().length]; - if (arguments.length > 0) { - MethodParameter methodParameter = MethodParameter.forExecutable(method, 0); - if (methodParameter.hasParameterAnnotation(Input.class)) { - Input methodAnnotation = methodParameter - .getParameterAnnotation(Input.class); - methodAnnotatedInboundName = methodAnnotation.value(); - } - } - } - return methodAnnotatedInboundName; - } - - @SuppressWarnings("unchecked") - private void adaptStreamListenerResult(Object outboundKStream, Object targetBean) { - for (StreamListenerResultAdapter streamListenerResultAdapter : this.streamListenerResultAdapters) { - if (streamListenerResultAdapter.supports( - outboundKStream.getClass(), targetBean.getClass())) { - streamListenerResultAdapter.adapt(outboundKStream, - targetBean); - break; - } - } - } - - @Override - @SuppressWarnings({"unchecked"}) - public Object[] adaptAndRetrieveInboundArguments(Method method, String inboundName, - ApplicationContext applicationContext, - StreamListenerParameterAdapter... adapters) { - Object[] arguments = new Object[method.getParameterTypes().length]; - for (int parameterIndex = 0; parameterIndex < arguments.length; parameterIndex++) { - MethodParameter methodParameter = MethodParameter.forExecutable(method, - parameterIndex); - Class parameterType = methodParameter.getParameterType(); - Object targetReferenceValue = null; - if (methodParameter.hasParameterAnnotation(Input.class)) { - targetReferenceValue = AnnotationUtils - .getValue(methodParameter.getParameterAnnotation(Input.class)); - Input methodAnnotation = methodParameter - .getParameterAnnotation(Input.class); - inboundName = methodAnnotation.value(); - } - else if (arguments.length == 1 && StringUtils.hasText(inboundName)) { - targetReferenceValue = inboundName; - } - if (targetReferenceValue != null) { - Assert.isInstanceOf(String.class, targetReferenceValue, - "Annotation value must be a String"); - Object targetBean = applicationContext - .getBean((String) targetReferenceValue); - BindingProperties bindingProperties = this.bindingServiceProperties - .getBindingProperties(inboundName); - // Retrieve the StreamsConfig created for this method if available. - // Otherwise, create the StreamsBuilderFactory and get the underlying - // config. - if (!this.methodStreamsBuilderFactoryBeanMap.containsKey(method)) { - StreamsBuilderFactoryBean streamsBuilderFactoryBean = buildStreamsBuilderAndRetrieveConfig(method.getDeclaringClass().getSimpleName() + "-" + method.getName(), - applicationContext, - inboundName, null, customizer, this.environment, bindingProperties); - this.methodStreamsBuilderFactoryBeanMap.put(method, streamsBuilderFactoryBean); - } - try { - StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.methodStreamsBuilderFactoryBeanMap - .get(method); - StreamsBuilder streamsBuilder = streamsBuilderFactoryBean.getObject(); - final String applicationId = streamsBuilderFactoryBean.getStreamsConfiguration().getProperty(StreamsConfig.APPLICATION_ID_CONFIG); - KafkaStreamsConsumerProperties extendedConsumerProperties = this.kafkaStreamsExtendedBindingProperties - .getExtendedConsumerProperties(inboundName); - extendedConsumerProperties.setApplicationId(applicationId); - // get state store spec - KafkaStreamsStateStoreProperties spec = buildStateStoreSpec(method); - - Serde keySerde = this.keyValueSerdeResolver - .getInboundKeySerde(extendedConsumerProperties, ResolvableType.forMethodParameter(methodParameter)); - LOG.info("Key Serde used for " + targetReferenceValue + ": " + keySerde.getClass().getName()); - - Serde valueSerde = bindingServiceProperties.getConsumerProperties(inboundName).isUseNativeDecoding() ? - getValueSerde(inboundName, extendedConsumerProperties, ResolvableType.forMethodParameter(methodParameter)) : Serdes.ByteArray(); - LOG.info("Value Serde used for " + targetReferenceValue + ": " + valueSerde.getClass().getName()); - - Topology.AutoOffsetReset autoOffsetReset = getAutoOffsetReset(inboundName, extendedConsumerProperties); - - if (parameterType.isAssignableFrom(KStream.class)) { - KStream stream = getkStream(inboundName, spec, - bindingProperties, extendedConsumerProperties, streamsBuilder, keySerde, valueSerde, - autoOffsetReset, parameterIndex == 0); - KStreamBoundElementFactory.KStreamWrapper kStreamWrapper = (KStreamBoundElementFactory.KStreamWrapper) targetBean; - // wrap the proxy created during the initial target type binding - // with real object (KStream) - kStreamWrapper.wrap((KStream) stream); - this.kafkaStreamsBindingInformationCatalogue.addKeySerde(stream, keySerde); - BindingProperties bindingProperties1 = this.kafkaStreamsBindingInformationCatalogue.getBindingProperties().get(kStreamWrapper); - this.kafkaStreamsBindingInformationCatalogue.registerBindingProperties(stream, bindingProperties1); - - this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding(inboundName, streamsBuilderFactoryBean); - this.kafkaStreamsBindingInformationCatalogue.addConsumerPropertiesPerSbfb(streamsBuilderFactoryBean, - bindingServiceProperties.getConsumerProperties(inboundName)); - - for (StreamListenerParameterAdapter streamListenerParameterAdapter : adapters) { - if (streamListenerParameterAdapter.supports(stream.getClass(), - methodParameter)) { - arguments[parameterIndex] = streamListenerParameterAdapter - .adapt(stream, methodParameter); - break; - } - } - if (arguments[parameterIndex] == null - && parameterType.isAssignableFrom(stream.getClass())) { - arguments[parameterIndex] = stream; - } - Assert.notNull(arguments[parameterIndex], - "Cannot convert argument " + parameterIndex + " of " - + method + "from " + stream.getClass() + " to " - + parameterType); - } - else { - handleKTableGlobalKTableInputs(arguments, parameterIndex, inboundName, parameterType, targetBean, streamsBuilderFactoryBean, - streamsBuilder, extendedConsumerProperties, keySerde, valueSerde, autoOffsetReset, parameterIndex == 0); - } - } - catch (Exception ex) { - throw new IllegalStateException(ex); - } - } - else { - throw new IllegalStateException( - StreamListenerErrorMessages.INVALID_DECLARATIVE_METHOD_PARAMETERS); - } - } - return arguments; - } - - private StoreBuilder buildStateStore(KafkaStreamsStateStoreProperties spec) { - try { - - Serde keySerde = this.keyValueSerdeResolver - .getStateStoreKeySerde(spec.getKeySerdeString()); - Serde valueSerde = this.keyValueSerdeResolver - .getStateStoreValueSerde(spec.getValueSerdeString()); - StoreBuilder builder; - switch (spec.getType()) { - case KEYVALUE: - builder = Stores.keyValueStoreBuilder( - Stores.persistentKeyValueStore(spec.getName()), keySerde, - valueSerde); - break; - case WINDOW: - builder = Stores - .windowStoreBuilder( - Stores.persistentWindowStore(spec.getName(), - Duration.ofMillis(spec.getRetention()), Duration.ofMillis(3), false), - keySerde, valueSerde); - break; - case SESSION: - builder = Stores.sessionStoreBuilder(Stores.persistentSessionStore( - spec.getName(), Duration.ofMillis(spec.getRetention())), keySerde, valueSerde); - break; - default: - throw new UnsupportedOperationException( - "state store type (" + spec.getType() + ") is not supported!"); - } - if (spec.isCacheEnabled()) { - builder = builder.withCachingEnabled(); - } - if (spec.isLoggingDisabled()) { - builder = builder.withLoggingDisabled(); - } - return builder; - } - catch (Exception ex) { - LOG.error("failed to build state store exception : " + ex); - throw ex; - } - } - - private KStream getkStream(String inboundName, - KafkaStreamsStateStoreProperties storeSpec, - BindingProperties bindingProperties, - KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, StreamsBuilder streamsBuilder, - Serde keySerde, Serde valueSerde, - Topology.AutoOffsetReset autoOffsetReset, boolean firstBuild) { - if (storeSpec != null) { - StoreBuilder storeBuilder = buildStateStore(storeSpec); - streamsBuilder.addStateStore(storeBuilder); - if (LOG.isInfoEnabled()) { - LOG.info("state store " + storeBuilder.name() + " added to topology"); - } - } - return getKStream(inboundName, bindingProperties, kafkaStreamsConsumerProperties, streamsBuilder, - keySerde, valueSerde, autoOffsetReset, firstBuild); - } - - private void validateStreamListenerMethod(StreamListener streamListener, - Method method, String[] methodAnnotatedOutboundNames) { - String methodAnnotatedInboundName = streamListener.value(); - if (methodAnnotatedOutboundNames != null) { - for (String s : methodAnnotatedOutboundNames) { - if (StringUtils.hasText(s)) { - Assert.isTrue(isDeclarativeOutput(method, s), - "Method must be declarative"); - } - } - } - if (StringUtils.hasText(methodAnnotatedInboundName)) { - int methodArgumentsLength = method.getParameterTypes().length; - - for (int parameterIndex = 0; parameterIndex < methodArgumentsLength; parameterIndex++) { - MethodParameter methodParameter = MethodParameter.forExecutable(method, - parameterIndex); - Assert.isTrue( - isDeclarativeInput(methodAnnotatedInboundName, methodParameter), - "Method must be declarative"); - } - } - } - - @SuppressWarnings("unchecked") - private boolean isDeclarativeOutput(Method m, String targetBeanName) { - boolean declarative; - Class returnType = m.getReturnType(); - if (returnType.isArray()) { - Class targetBeanClass = this.applicationContext.getType(targetBeanName); - declarative = this.streamListenerResultAdapters.stream() - .anyMatch((slpa) -> slpa.supports(returnType.getComponentType(), - targetBeanClass)); - return declarative; - } - Class targetBeanClass = this.applicationContext.getType(targetBeanName); - declarative = this.streamListenerResultAdapters.stream() - .anyMatch((slpa) -> slpa.supports(returnType, targetBeanClass)); - return declarative; - } - - @SuppressWarnings("unchecked") - private boolean isDeclarativeInput(String targetBeanName, - MethodParameter methodParameter) { - if (!methodParameter.getParameterType().isAssignableFrom(Object.class) - && this.applicationContext.containsBean(targetBeanName)) { - Class targetBeanClass = this.applicationContext.getType(targetBeanName); - if (targetBeanClass != null) { - boolean supports = KafkaStreamsBinderUtils.supportsKStream(methodParameter, targetBeanClass); - if (!supports) { - supports = KTable.class.isAssignableFrom(targetBeanClass) - && KTable.class.isAssignableFrom(methodParameter.getParameterType()); - if (!supports) { - supports = GlobalKTable.class.isAssignableFrom(targetBeanClass) - && GlobalKTable.class.isAssignableFrom(methodParameter.getParameterType()); - } - } - return supports; - } - } - return false; - } - - private static String[] getOutboundBindingTargetNames(Method method) { - SendTo sendTo = AnnotationUtils.findAnnotation(method, SendTo.class); - if (sendTo != null) { - Assert.isTrue(!ObjectUtils.isEmpty(sendTo.value()), - StreamListenerErrorMessages.ATLEAST_ONE_OUTPUT); - Assert.isTrue(sendTo.value().length >= 1, - "At least one outbound destination need to be provided."); - return sendTo.value(); - } - return null; - } - - @SuppressWarnings({"unchecked"}) - private KafkaStreamsStateStoreProperties buildStateStoreSpec(Method method) { - if (!this.registeredStoresPerMethod.containsKey(method)) { - KafkaStreamsStateStore spec = AnnotationUtils.findAnnotation(method, - KafkaStreamsStateStore.class); - if (spec != null) { - Assert.isTrue(!ObjectUtils.isEmpty(spec.name()), "name cannot be empty"); - Assert.isTrue(spec.name().length() >= 1, "name cannot be empty."); - this.registeredStoresPerMethod.put(method, new ArrayList<>()); - this.registeredStoresPerMethod.get(method).add(spec.name()); - KafkaStreamsStateStoreProperties props = new KafkaStreamsStateStoreProperties(); - props.setName(spec.name()); - props.setType(spec.type()); - props.setLength(spec.lengthMs()); - props.setKeySerdeString(spec.keySerde()); - props.setRetention(spec.retentionMs()); - props.setValueSerdeString(spec.valueSerde()); - props.setCacheEnabled(spec.cache()); - props.setLoggingDisabled(!spec.logging()); - return props; - } - } - return null; - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsProcessor.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsProcessor.java deleted file mode 100644 index 36df1f43f..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsProcessor.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2017-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.annotations; - -import org.apache.kafka.streams.kstream.KStream; - -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; - -/** - * Bindable interface for {@link KStream} input and output. - * - * This interface can be used as a bindable interface with - * {@link org.springframework.cloud.stream.annotation.EnableBinding} when both input and - * output types are single KStream. In other scenarios where multiple types are required, - * other similar bindable interfaces can be created and used. For example, there are cases - * in which multiple KStreams are required on the outbound in the case of KStream - * branching or multiple input types are required either in the form of multiple KStreams - * and a combination of KStreams and KTables. In those cases, new bindable interfaces - * compatible with the requirements must be created. Here are some examples. - * - * 

- *     interface KStreamBranchProcessor {
- *         @Input("input")
- *         KStream<?, ?> input();
- *
- *         @Output("output-1")
- *         KStream<?, ?> output1();
- *
- *         @Output("output-2")
- *         KStream<?, ?> output2();
- *
- *         @Output("output-3")
- *         KStream<?, ?> output3();
- *
- *         ......
- *
- *     }
- *
- * - * 
- *     interface KStreamKtableProcessor {
- *         @Input("input-1")
- *         KStream<?, ?> input1();
- *
- *         @Input("input-2")
- *         KTable<?, ?> input2();
- *
- *         @Output("output")
- *         KStream<?, ?> output();
- *
- *         ......
- *
- *     }
- *
- * - * @author Marius Bogoevici - * @author Soby Chacko - */ -public interface KafkaStreamsProcessor { - - /** - * Input binding. - * @return {@link Input} binding for {@link KStream} type. - */ - @Input("input") - KStream input(); - - /** - * Output binding. - * @return {@link Output} binding for {@link KStream} type. - */ - @Output("output") - KStream output(); - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsStateStore.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsStateStore.java deleted file mode 100644 index a24cda301..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsStateStore.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.annotations; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsStateStoreProperties; - -/** - * Interface for Kafka Stream state store. - * - * This interface can be used to inject a state store specification into KStream building - * process so that the desired store can be built by StreamBuilder and added to topology - * for later use by processors. This is particularly useful when need to combine stream - * DSL with low level processor APIs. In those cases, if a writable state store is desired - * in processors, it needs to be created using this annotation. Here is the example. - * - * 
- *     @StreamListener("input")
- *     @KafkaStreamsStateStore(name="mystate", type= KafkaStreamsStateStoreProperties.StoreType.WINDOW,
- *     								size=300000)
- *	   public void process(KStream<Object, Product> input) {
- *         ......
- *     }
- *
- * - * With that, you should be able to read/write this state store in your - * processor/transformer code. - * - * 
- * 		new Processor<Object, Product>() {
- * 			WindowStore<Object, String> state;
- * 			@Override
- *			public void init(ProcessorContext processorContext) {
- *			state = (WindowStore)processorContext.getStateStore("mystate");
- *				......
- *			}
- *		}
- *
- * - * @author Lei Chen - */ - -@Target({ ElementType.TYPE, ElementType.METHOD, ElementType.ANNOTATION_TYPE }) -@Retention(RetentionPolicy.RUNTIME) - -public @interface KafkaStreamsStateStore { - - /** - * Provides name of the state store. - * @return name of state store. - */ - String name() default ""; - - /** - * State store type. - * @return {@link KafkaStreamsStateStoreProperties.StoreType} of state store. - */ - KafkaStreamsStateStoreProperties.StoreType type() default KafkaStreamsStateStoreProperties.StoreType.KEYVALUE; - - /** - * Serde used for key. - * @return key serde of state store. - */ - String keySerde() default "org.apache.kafka.common.serialization.Serdes$StringSerde"; - - /** - * Serde used for value. - * @return value serde of state store. - */ - String valueSerde() default "org.apache.kafka.common.serialization.Serdes$StringSerde"; - - /** - * Length in milli-second of Windowed store window. - * @return length in milli-second of window(for windowed store). - */ - long lengthMs() default 0; - - /** - * Retention period for Windowed store windows. - * @return the maximum period of time in milli-second to keep each window in this - * store(for windowed store). - */ - long retentionMs() default 0; - - /** - * Whether catching is enabled or not. - * @return whether caching should be enabled on the created store. - */ - boolean cache() default false; - - /** - * Whether logging is enabled or not. - * @return whether logging should be enabled on the created store. - */ - boolean logging() default true; - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsStateStoreProperties.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsStateStoreProperties.java deleted file mode 100644 index c51cc6120..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsStateStoreProperties.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.properties; - -/** - * Properties for Kafka Streams state store. - * - * @author Lei Chen - */ -public class KafkaStreamsStateStoreProperties { - - /** - * Enumeration for store type. - */ - public enum StoreType { - - /** - * Key value store. - */ - KEYVALUE("keyvalue"), - /** - * Window store. - */ - WINDOW("window"), - /** - * Session store. - */ - SESSION("session"); - - private final String type; - - StoreType(final String type) { - this.type = type; - } - - @Override - public String toString() { - return this.type; - } - - } - - /** - * Name for this state store. - */ - private String name; - - /** - * Type for this state store. - */ - private StoreType type; - - /** - * Size/length of this state store in ms. Only applicable for window store. - */ - private long length; - - /** - * Retention period for this state store in ms. - */ - private long retention; - - /** - * Key serde class specified per state store. - */ - private String keySerdeString; - - /** - * Value serde class specified per state store. - */ - private String valueSerdeString; - - /** - * Whether caching is enabled on this state store. - */ - private boolean cacheEnabled; - - /** - * Whether logging is enabled on this state store. - */ - private boolean loggingDisabled; - - public String getName() { - return this.name; - } - - public void setName(String name) { - this.name = name; - } - - public StoreType getType() { - return this.type; - } - - public void setType(StoreType type) { - this.type = type; - } - - public long getLength() { - return this.length; - } - - public void setLength(long length) { - this.length = length; - } - - public long getRetention() { - return this.retention; - } - - public void setRetention(long retention) { - this.retention = retention; - } - - public String getKeySerdeString() { - return this.keySerdeString; - } - - public void setKeySerdeString(String keySerdeString) { - this.keySerdeString = keySerdeString; - } - - public String getValueSerdeString() { - return this.valueSerdeString; - } - - public void setValueSerdeString(String valueSerdeString) { - this.valueSerdeString = valueSerdeString; - } - - public boolean isCacheEnabled() { - return this.cacheEnabled; - } - - public void setCacheEnabled(boolean cacheEnabled) { - this.cacheEnabled = cacheEnabled; - } - - public boolean isLoggingDisabled() { - return this.loggingDisabled; - } - - public void setLoggingDisabled(boolean loggingDisabled) { - this.loggingDisabled = loggingDisabled; - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CompositeNonNativeSerde.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CompositeNonNativeSerde.java deleted file mode 100644 index a7b0300ac..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CompositeNonNativeSerde.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.serde; - -import org.springframework.messaging.converter.CompositeMessageConverter; - -/** - * This class provides the same functionality as {@link MessageConverterDelegateSerde} and is deprecated. - * It is kept for backward compatibility reasons and will be removed in version 3.1 - * - * @author Soby Chacko - * @since 2.1 - * - * @deprecated in favor of {@link MessageConverterDelegateSerde} - */ -@Deprecated -public class CompositeNonNativeSerde extends MessageConverterDelegateSerde { - - public CompositeNonNativeSerde(CompositeMessageConverter compositeMessageConverter) { - super(compositeMessageConverter); - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/MessageConverterDelegateSerde.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/MessageConverterDelegateSerde.java deleted file mode 100644 index a64e28695..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/MessageConverterDelegateSerde.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright 2019-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.serde; - -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; - -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serializer; - -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.converter.CompositeMessageConverter; -import org.springframework.messaging.converter.MessageConverter; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.util.Assert; -import org.springframework.util.MimeType; -import org.springframework.util.MimeTypeUtils; - -/** - * A {@link Serde} implementation that wraps the list of {@link MessageConverter}s from - * {@link CompositeMessageConverter}. - * - * The primary motivation for this class is to provide an avro based {@link Serde} that is - * compatible with the schema registry that Spring Cloud Stream provides. When using the - * schema registry support from Spring Cloud Stream in a Kafka Streams binder based - * application, the applications can deserialize the incoming Kafka Streams records using - * the built in Avro {@link MessageConverter}. However, this same message conversion - * approach will not work downstream in other operations in the topology for Kafka Streams - * as some of them needs a {@link Serde} instance that can talk to the Spring Cloud Stream - * provided Schema Registry. This implementation will solve that problem. - * - * Only Avro and JSON based converters are exposed as binder provided {@link Serde} - * implementations currently. - * - * Users of this class must call the - * {@link MessageConverterDelegateSerde#configure(Map, boolean)} method to configure the - * {@link Serde} object. At the very least the configuration map must include a key called - * "valueClass" to indicate the type of the target object for deserialization. If any - * other content type other than JSON is needed (only Avro is available now other than - * JSON), that needs to be included in the configuration map with the key "contentType". - * For example, - * - * 
- * Map<String, Object> config = new HashMap<>();
- * config.put("valueClass", Foo.class);
- * config.put("contentType", "application/avro");
- *
- * - * Then use the above map when calling the configure method. - * - * This class is only intended to be used when writing a Spring Cloud Stream Kafka Streams - * application that uses Spring Cloud Stream schema registry for schema evolution. - * - * An instance of this class is provided as a bean by the binder configuration and - * typically the applications can autowire that bean. This is the expected usage pattern - * of this class. - * - * @param type of the object to marshall - * @author Soby Chacko - * @since 3.0 - * @deprecated in favor of other schema registry providers instead of Spring Cloud Schema Registry. See its motivation above. - */ -@Deprecated -public class MessageConverterDelegateSerde implements Serde { - - private static final String VALUE_CLASS_HEADER = "valueClass"; - - private static final String AVRO_FORMAT = "avro"; - - private static final MimeType DEFAULT_AVRO_MIME_TYPE = new MimeType("application", - "*+" + AVRO_FORMAT); - - private final MessageConverterDelegateDeserializer messageConverterDelegateDeserializer; - - private final MessageConverterDelegateSerializer messageConverterDelegateSerializer; - - public MessageConverterDelegateSerde( - CompositeMessageConverter compositeMessageConverter) { - this.messageConverterDelegateDeserializer = new MessageConverterDelegateDeserializer<>( - compositeMessageConverter); - this.messageConverterDelegateSerializer = new MessageConverterDelegateSerializer<>( - compositeMessageConverter); - } - - @Override - public void configure(Map configs, boolean isKey) { - this.messageConverterDelegateDeserializer.configure(configs, isKey); - this.messageConverterDelegateSerializer.configure(configs, isKey); - } - - @Override - public void close() { - // No-op - } - - @Override - public Serializer serializer() { - return this.messageConverterDelegateSerializer; - } - - @Override - public Deserializer deserializer() { - return this.messageConverterDelegateDeserializer; - } - - private static MimeType resolveMimeType(Map configs) { - if (configs.containsKey(MessageHeaders.CONTENT_TYPE)) { - String contentType = (String) configs.get(MessageHeaders.CONTENT_TYPE); - if (DEFAULT_AVRO_MIME_TYPE.equals(MimeTypeUtils.parseMimeType(contentType))) { - return DEFAULT_AVRO_MIME_TYPE; - } - else if (contentType.contains("avro")) { - return MimeTypeUtils.parseMimeType("application/avro"); - } - else { - return new MimeType("application", "json", StandardCharsets.UTF_8); - } - } - else { - return new MimeType("application", "json", StandardCharsets.UTF_8); - } - } - - /** - * Custom {@link Deserializer} that uses the {@link org.springframework.cloud.stream.converter.CompositeMessageConverterFactory}. - * - * @param parameterized target type for deserialization - */ - private static class MessageConverterDelegateDeserializer implements Deserializer { - - private final MessageConverter messageConverter; - - private MimeType mimeType; - - private Class valueClass; - - MessageConverterDelegateDeserializer( - CompositeMessageConverter compositeMessageConverter) { - this.messageConverter = compositeMessageConverter; - } - - @Override - public void configure(Map configs, boolean isKey) { - Assert.isTrue(configs.containsKey(VALUE_CLASS_HEADER), - "Deserializers must provide a configuration for valueClass."); - final Object valueClass = configs.get(VALUE_CLASS_HEADER); - Assert.isTrue(valueClass instanceof Class, - "Deserializers must provide a valid value for valueClass."); - this.valueClass = (Class) valueClass; - this.mimeType = resolveMimeType(configs); - } - - @SuppressWarnings("unchecked") - @Override - public U deserialize(String topic, byte[] data) { - Message message = MessageBuilder.withPayload(data) - .setHeader(MessageHeaders.CONTENT_TYPE, this.mimeType.toString()) - .build(); - U messageConverted = (U) this.messageConverter.fromMessage(message, - this.valueClass); - Assert.notNull(messageConverted, "Deserialization failed."); - return messageConverted; - } - - @Override - public void close() { - // No-op - } - - } - - /** - * Custom {@link Serializer} that uses the {@link org.springframework.cloud.stream.converter.CompositeMessageConverterFactory}. - * - * @param parameterized type for serialization - */ - private static class MessageConverterDelegateSerializer implements Serializer { - - private final MessageConverter messageConverter; - - private MimeType mimeType; - - MessageConverterDelegateSerializer( - CompositeMessageConverter compositeMessageConverter) { - this.messageConverter = compositeMessageConverter; - } - - @Override - public void configure(Map configs, boolean isKey) { - this.mimeType = resolveMimeType(configs); - } - - @Override - public byte[] serialize(String topic, V data) { - Message message = MessageBuilder.withPayload(data).build(); - Map headers = new HashMap<>(message.getHeaders()); - headers.put(MessageHeaders.CONTENT_TYPE, this.mimeType.toString()); - MessageHeaders messageHeaders = new MessageHeaders(headers); - final Object payload = this.messageConverter - .toMessage(message.getPayload(), messageHeaders).getPayload(); - return (byte[]) payload; - } - - @Override - public void close() { - // No-op - } - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/resources/avro/sensor.avsc b/spring-cloud-stream-binder-kafka-streams/src/test/resources/avro/sensor.avsc deleted file mode 100644 index c0e060d3d..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/resources/avro/sensor.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "namespace" : "com.example", - "type" : "record", - "name" : "Sensor", - "fields" : [ - {"name":"id","type":"string"}, - {"name":"temperature", "type":"float", "default":0.0}, - {"name":"acceleration", "type":"float","default":0.0}, - {"name":"velocity","type":"float","default":0.0} - ] -} From 486469da51dfd096f4bcf202a5672eb4fbc630a7 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Thu, 11 Nov 2021 16:33:45 -0500 Subject: [PATCH 03/19] Kafka binder test migration - EnableBinding to functional --- ...fkaStreamsBindingInformationCatalogue.java | 2 +- .../KafkaStreamsFunctionProcessor.java | 3 +- .../streams/StreamsBuilderFactoryManager.java | 3 +- .../integration/KafkaBinderActuatorTests.java | 52 ++++++++------- .../KafkaBinderExtendedPropertiesTest.java | 55 +++++----------- .../integration/KafkaNullConverterTest.java | 58 ++++++++--------- .../ProducerOnlyTransactionTests.java | 19 +++--- ...eKafkaBinderTopicPropertiesUpdateTest.java | 33 +++------- .../ConsumerProducerTransactionTests.java | 65 +++++++++---------- 9 files changed, 124 insertions(+), 166 deletions(-) diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java index a8dc9b8b7..c12b77d07 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java @@ -41,7 +41,7 @@ import org.springframework.util.CollectionUtils; * A catalogue that provides binding information for Kafka Streams target types such as * KStream. It also keeps a catalogue for the underlying {@link StreamsBuilderFactoryBean} * and {@link StreamsConfig} associated with various - * {@link org.springframework.cloud.stream.annotation.StreamListener} methods in the + * Kafka Streams functions in the * {@link org.springframework.context.ApplicationContext}. * * @author Soby Chacko diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionProcessor.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionProcessor.java index c91953715..ff0286c1a 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionProcessor.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionProcessor.java @@ -51,7 +51,6 @@ import org.springframework.cloud.stream.binder.kafka.streams.function.KafkaStrea import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; -import org.springframework.cloud.stream.binding.StreamListenerErrorMessages; import org.springframework.cloud.stream.config.BindingProperties; import org.springframework.cloud.stream.config.BindingServiceProperties; import org.springframework.cloud.stream.function.FunctionConstants; @@ -562,7 +561,7 @@ public class KafkaStreamsFunctionProcessor extends AbstractKafkaStreamsBinderPro } } else { - throw new IllegalStateException(StreamListenerErrorMessages.INVALID_DECLARATIVE_METHOD_PARAMETERS); + //throw new IllegalStateException(StreamListenerErrorMessages.INVALID_DECLARATIVE_METHOD_PARAMETERS); } } return arguments; diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/StreamsBuilderFactoryManager.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/StreamsBuilderFactoryManager.java index 9f2bbf0f6..c0fa13e49 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/StreamsBuilderFactoryManager.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/StreamsBuilderFactoryManager.java @@ -39,8 +39,7 @@ import org.springframework.kafka.streams.KafkaStreamsMicrometerListener; * This {@link SmartLifecycle} class ensures that the bean created from it is started very * late through the bootstrap process by setting the phase value closer to * Integer.MAX_VALUE. This is to guarantee that the {@link StreamsBuilderFactoryBean} on a - * {@link org.springframework.cloud.stream.annotation.StreamListener} method with multiple - * bindings is only started after all the binding phases have completed successfully. + * function with multiple bindings is only started after all the binding phases have completed successfully. * * @author Soby Chacko */ diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderActuatorTests.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderActuatorTests.java index f722dc510..ce5e1d105 100644 --- a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderActuatorTests.java +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderActuatorTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 the original author or authors. + * Copyright 2018-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,12 +18,14 @@ package org.springframework.cloud.stream.binder.kafka.integration; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.MeterBinder; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; @@ -33,19 +35,14 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.FilteredClassLoader; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.StreamListener; import org.springframework.cloud.stream.binder.Binding; -import org.springframework.cloud.stream.binder.PollableMessageSource; import org.springframework.cloud.stream.binding.BindingService; import org.springframework.cloud.stream.config.ConsumerEndpointCustomizer; import org.springframework.cloud.stream.config.ListenerContainerCustomizer; import org.springframework.cloud.stream.config.MessageSourceCustomizer; import org.springframework.cloud.stream.config.ProducerMessageHandlerCustomizer; -import org.springframework.cloud.stream.messaging.Processor; -import org.springframework.cloud.stream.messaging.Sink; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; import org.springframework.integration.kafka.inbound.KafkaMessageSource; import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler; @@ -63,13 +60,18 @@ import static org.assertj.core.api.Assertions.assertThat; * @author Oleg Zhurakousky * @author Jon Schneider * @author Gary Russell + * @author Soby Chacko * * @since 2.0 */ @RunWith(SpringRunner.class) // @checkstyle:off -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = "spring.cloud.stream.bindings.input.group=" - + KafkaBinderActuatorTests.TEST_CONSUMER_GROUP) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, + properties = { + "spring.cloud.stream.bindings.input.group=" + KafkaBinderActuatorTests.TEST_CONSUMER_GROUP, + "spring.cloud.stream.function.bindings.process-in-0=input", + "spring.cloud.stream.pollable-source=input"} +) // @checkstyle:on @DirtiesContext public class KafkaBinderActuatorTests { @@ -100,17 +102,22 @@ public class KafkaBinderActuatorTests { @Test public void testKafkaBinderMetricsExposed() { - this.kafkaTemplate.send(Sink.INPUT, null, "foo".getBytes()); + this.kafkaTemplate.send("input", null, "foo".getBytes()); this.kafkaTemplate.flush(); assertThat(this.meterRegistry.get("spring.cloud.stream.binder.kafka.offset") - .tag("group", TEST_CONSUMER_GROUP).tag("topic", Sink.INPUT).gauge() + .tag("group", TEST_CONSUMER_GROUP).tag("topic", "input").gauge() .value()).isGreaterThan(0); } @Test + @Ignore public void testKafkaBinderMetricsWhenNoMicrometer() { new ApplicationContextRunner().withUserConfiguration(KafkaMetricsTestConfig.class) + .withPropertyValues( + "spring.cloud.stream.bindings.input.group", KafkaBinderActuatorTests.TEST_CONSUMER_GROUP, + "spring.cloud.stream.function.bindings.process-in-0", "input", + "spring.cloud.stream.pollable-source", "input") .withClassLoader(new FilteredClassLoader("io.micrometer.core")) .run(context -> { assertThat(context.getBeanNamesForType(MeterRegistry.class)) @@ -148,8 +155,8 @@ public class KafkaBinderActuatorTests { }); } - @EnableBinding({ Processor.class, PMS.class }) @EnableAutoConfiguration + @Configuration public static class KafkaMetricsTestConfig { @Bean @@ -172,19 +179,18 @@ public class KafkaBinderActuatorTests { return (handler, destinationName) -> handler.setBeanName("setByCustomizer:" + destinationName); } - @StreamListener(Sink.INPUT) - public void process(@SuppressWarnings("unused") String payload) throws InterruptedException { + @Bean + public Consumer process() { // Artificial slow listener to emulate consumer lag - Thread.sleep(1000); + return s -> { + try { + Thread.sleep(1000); + } + catch (InterruptedException e) { + //no-op + } + }; } } - - public interface PMS { - - @Input - PollableMessageSource source(); - - } - } diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderExtendedPropertiesTest.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderExtendedPropertiesTest.java index db2064c0d..64b5a79b5 100644 --- a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderExtendedPropertiesTest.java +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderExtendedPropertiesTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 the original author or authors. + * Copyright 2018-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.common.TopicPartition; @@ -33,10 +34,6 @@ import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.cloud.stream.annotation.StreamListener; import org.springframework.cloud.stream.binder.Binder; import org.springframework.cloud.stream.binder.BinderFactory; import org.springframework.cloud.stream.binder.ConsumerProperties; @@ -47,10 +44,9 @@ import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerPro import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.SubscribableChannel; -import org.springframework.messaging.handler.annotation.SendTo; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.junit4.SpringRunner; @@ -62,6 +58,11 @@ import static org.assertj.core.api.Assertions.assertThat; */ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { + "spring.cloud.stream.function.definition=process;processCustom", + "spring.cloud.stream.function.bindings.process-in-0=standard-in", + "spring.cloud.stream.function.bindings.process-out-0=standard-out", + "spring.cloud.stream.function.bindings.processCustom-in-0=custom-in", + "spring.cloud.stream.function.bindings.processCustom-out-0=custom-out", "spring.cloud.stream.kafka.bindings.standard-out.producer.configuration.key.serializer=FooSerializer.class", "spring.cloud.stream.kafka.default.producer.configuration.key.serializer=BarSerializer.class", "spring.cloud.stream.kafka.default.producer.configuration.value.serializer=BarSerializer.class", @@ -167,22 +168,19 @@ public class KafkaBinderExtendedPropertiesTest { Boolean.TRUE); } - @EnableBinding(CustomBindingForExtendedPropertyTesting.class) @EnableAutoConfiguration + @Configuration public static class KafkaMetricsTestConfig { - @StreamListener("standard-in") - @SendTo("standard-out") - public String process(String payload) { - return payload; - } - - @StreamListener("custom-in") - @SendTo("custom-out") - public String processCustom(String payload) { - return payload; + @Bean + public Function process() { + return payload -> payload; } + @Bean + public Function processCustom() { + return payload -> payload; + } @Bean public RebalanceListener rebalanceListener() { return new RebalanceListener(); @@ -190,22 +188,6 @@ public class KafkaBinderExtendedPropertiesTest { } - interface CustomBindingForExtendedPropertyTesting { - - @Input("standard-in") - SubscribableChannel standardIn(); - - @Output("standard-out") - MessageChannel standardOut(); - - @Input("custom-in") - SubscribableChannel customIn(); - - @Output("custom-out") - MessageChannel customOut(); - - } - public static class RebalanceListener implements KafkaBindingRebalanceListener { private final Map bindings = new HashMap<>(); @@ -215,23 +197,18 @@ public class KafkaBinderExtendedPropertiesTest { @Override public void onPartitionsRevokedBeforeCommit(String bindingName, Consumer consumer, Collection partitions) { - } @Override public void onPartitionsRevokedAfterCommit(String bindingName, Consumer consumer, Collection partitions) { - } @Override public void onPartitionsAssigned(String bindingName, Consumer consumer, Collection partitions, boolean initial) { - this.bindings.put(bindingName, initial); this.latch.countDown(); } - } - } diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaNullConverterTest.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaNullConverterTest.java index 3015b10f8..f796069e5 100644 --- a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaNullConverterTest.java +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaNullConverterTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2017 the original author or authors. + * Copyright 2016-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,21 +18,21 @@ package org.springframework.cloud.stream.binder.kafka.integration; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.support.KafkaNull; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; @@ -47,21 +47,19 @@ import static org.assertj.core.api.Assertions.assertThat; /** * @author Aldo Sinanaj * @author Gary Russell + * @author Soby Chacko */ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { - "spring.kafka.consumer.auto-offset-reset=earliest" }) + "spring.kafka.consumer.auto-offset-reset=earliest", + "spring.cloud.stream.function.bindings.inputListen-in-0=kafkaNullInput"}) @DirtiesContext -@Ignore public class KafkaNullConverterTest { private static final String KAFKA_BROKERS_PROPERTY = "spring.kafka.bootstrap-servers"; @Autowired - private MessageChannel kafkaNullOutput; - - @Autowired - private MessageChannel kafkaNullInput; + private ApplicationContext context; @Autowired private KafkaNullConverterTestConfig config; @@ -82,7 +80,9 @@ public class KafkaNullConverterTest { @Test public void testKafkaNullConverterOutput() throws InterruptedException { - this.kafkaNullOutput.send(new GenericMessage<>(KafkaNull.INSTANCE)); + final StreamBridge streamBridge = context.getBean(StreamBridge.class); + + streamBridge.send("kafkaNullOutput", new GenericMessage<>(KafkaNull.INSTANCE)); assertThat(this.config.countDownLatchOutput.await(10, TimeUnit.SECONDS)).isTrue(); assertThat(this.config.outputPayload).isNull(); @@ -90,14 +90,17 @@ public class KafkaNullConverterTest { @Test public void testKafkaNullConverterInput() throws InterruptedException { - this.kafkaNullInput.send(new GenericMessage<>(KafkaNull.INSTANCE)); + + final MessageChannel kafkaNullInput = context.getBean("kafkaNullInput", MessageChannel.class); + + kafkaNullInput.send(new GenericMessage<>(KafkaNull.INSTANCE)); assertThat(this.config.countDownLatchInput.await(10, TimeUnit.SECONDS)).isTrue(); assertThat(this.config.inputPayload).isNull(); } - @TestConfiguration - @EnableBinding(KafkaNullTestChannels.class) + @EnableAutoConfiguration + @Configuration public static class KafkaNullConverterTestConfig { final CountDownLatch countDownLatchOutput = new CountDownLatch(1); @@ -114,22 +117,13 @@ public class KafkaNullConverterTest { countDownLatchOutput.countDown(); } - @StreamListener("kafkaNullInput") - public void inputListen(@Payload(required = false) byte[] payload) { - this.inputPayload = payload; - countDownLatchInput.countDown(); + @Bean + public Consumer inputListen() { + return in -> { + this.inputPayload = in; + countDownLatchInput.countDown(); + }; } } - - public interface KafkaNullTestChannels { - - @Input - MessageChannel kafkaNullInput(); - - @Output - MessageChannel kafkaNullOutput(); - - } - } diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/ProducerOnlyTransactionTests.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/ProducerOnlyTransactionTests.java index 41e14f26d..861a7e152 100644 --- a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/ProducerOnlyTransactionTests.java +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/ProducerOnlyTransactionTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2019 the original author or authors. + * Copyright 2019-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,11 +35,12 @@ import org.springframework.beans.factory.BeanCreationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.stream.annotation.EnableBinding; import org.springframework.cloud.stream.binder.BinderFactory; import org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder; -import org.springframework.cloud.stream.messaging.Source; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.kafka.core.ProducerFactory; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; @@ -58,6 +59,7 @@ import static org.assertj.core.api.Assertions.assertThat; /** * @author Gary Russell + * @author Soby Chacko * @since 2.1.4 * */ @@ -80,7 +82,7 @@ public class ProducerOnlyTransactionTests { private Sender sender; @Autowired - private MessageChannel output; + private ApplicationContext context; @BeforeClass public static void setup() { @@ -95,7 +97,8 @@ public class ProducerOnlyTransactionTests { @Test public void testProducerTx() { - this.sender.DoInTransaction(this.output); + final StreamBridge streamBridge = context.getBean(StreamBridge.class); + this.sender.DoInTransaction(streamBridge); assertThat(this.sender.isInTx()).isTrue(); Map props = KafkaTestUtils.consumerProps("consumeTx", "false", embeddedKafka.getEmbeddedKafka()); @@ -109,9 +112,9 @@ public class ProducerOnlyTransactionTests { assertThat(record.value()).isEqualTo("foo".getBytes()); } - @EnableBinding(Source.class) @EnableAutoConfiguration @EnableTransactionManagement + @Configuration public static class Config { @Bean @@ -140,9 +143,9 @@ public class ProducerOnlyTransactionTests { private boolean isInTx; @Transactional - public void DoInTransaction(MessageChannel output) { + public void DoInTransaction(StreamBridge streamBridge) { this.isInTx = TransactionSynchronizationManager.isActualTransactionActive(); - output.send(new GenericMessage<>("foo")); + streamBridge.send("output", new GenericMessage<>("foo".getBytes())); } public boolean isInTx() { diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/BaseKafkaBinderTopicPropertiesUpdateTest.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/BaseKafkaBinderTopicPropertiesUpdateTest.java index bfaee22c1..72bec2af8 100644 --- a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/BaseKafkaBinderTopicPropertiesUpdateTest.java +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/BaseKafkaBinderTopicPropertiesUpdateTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 the original author or authors. + * Copyright 2018-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ package org.springframework.cloud.stream.binder.kafka.integration.topic.configs; +import java.util.function.Function; + import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -23,24 +25,21 @@ import org.junit.runner.RunWith; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.context.annotation.Bean; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; -import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.SubscribableChannel; -import org.springframework.messaging.handler.annotation.SendTo; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.junit4.SpringRunner; /** * @author Heiko Does + * @author Soby Chacko */ @RunWith(SpringRunner.class) @SpringBootTest( classes = BaseKafkaBinderTopicPropertiesUpdateTest.TopicAutoConfigsTestConfig.class, webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { + "spring.cloud.stream.function.bindings.process-in-0=standard-in", + "spring.cloud.stream.function.bindings.process-out-0=standard-out", "spring.cloud.stream.kafka.bindings.standard-out.producer.topic.properties.retention.ms=9001", "spring.cloud.stream.kafka.default.producer.topic.properties.retention.ms=-1", "spring.cloud.stream.kafka.bindings.standard-in.consumer.topic.properties.retention.ms=9001", @@ -65,24 +64,12 @@ public abstract class BaseKafkaBinderTopicPropertiesUpdateTest { System.clearProperty(KAFKA_BROKERS_PROPERTY); } - @EnableBinding(CustomBindingForTopicPropertiesUpdateTesting.class) @EnableAutoConfiguration public static class TopicAutoConfigsTestConfig { - @StreamListener("standard-in") - @SendTo("standard-out") - public String process(String payload) { - return payload; + @Bean + public Function process() { + return payload -> payload; } } - - interface CustomBindingForTopicPropertiesUpdateTesting { - - @Input("standard-in") - SubscribableChannel standardIn(); - - @Output("standard-out") - MessageChannel standardOut(); - } - } diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/ConsumerProducerTransactionTests.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/ConsumerProducerTransactionTests.java index 4f49b2061..dbf30a093 100644 --- a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/ConsumerProducerTransactionTests.java +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/ConsumerProducerTransactionTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2019 the original author or authors. + * Copyright 2019-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import kafka.server.KafkaConfig; import org.junit.AfterClass; @@ -33,13 +34,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.ApplicationRunner; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.cloud.stream.annotation.StreamListener; import org.springframework.cloud.stream.config.ListenerContainerCustomizer; -import org.springframework.cloud.stream.messaging.Processor; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.core.ProducerFactory; @@ -49,8 +47,6 @@ import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; import org.springframework.kafka.transaction.KafkaAwareTransactionManager; import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.SubscribableChannel; -import org.springframework.messaging.support.GenericMessage; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.util.backoff.FixedBackOff; @@ -61,6 +57,7 @@ import static org.mockito.Mockito.mock; /** * @author Gary Russell + * @author Soby Chacko * @since 3.0 * */ @@ -69,6 +66,11 @@ import static org.mockito.Mockito.mock; "spring.kafka.consumer.properties.isolation.level=read_committed", "spring.kafka.consumer.enable-auto-commit=false", "spring.kafka.consumer.auto-offset-reset=earliest", + "spring.cloud.function.definition=listenIn;listenIn2", + "spring.cloud.stream.function.bindings.listenIn-in-0=input", + "spring.cloud.stream.function.bindings.listenIn-out-0=output", + "spring.cloud.stream.function.bindings.listenIn2-in-0=input2", + "spring.cloud.stream.function.bindings.listenIn2-out-0=output2", "spring.cloud.stream.bindings.input.destination=consumer.producer.txIn", "spring.cloud.stream.bindings.input.group=consumer.producer.tx", "spring.cloud.stream.bindings.input.consumer.max-attempts=1", @@ -91,6 +93,9 @@ public class ConsumerProducerTransactionTests { @Autowired private Config config; + @Autowired + private ApplicationContext context; + @BeforeClass public static void setup() { System.setProperty(KAFKA_BROKERS_PROPERTY, @@ -115,26 +120,22 @@ public class ConsumerProducerTransactionTests { public void externalTM() { assertThat(this.config.input2Container.getContainerProperties().getTransactionManager()) .isSameAs(this.config.tm); - Object handler = KafkaTestUtils.getPropertyValue(this.config.output2, "dispatcher.handlers", Set.class) + final MessageChannel output2 = context.getBean("output2", MessageChannel.class); + + Object handler = KafkaTestUtils.getPropertyValue(output2, "dispatcher.handlers", Set.class) .iterator().next(); assertThat(KafkaTestUtils.getPropertyValue(handler, "delegate.kafkaTemplate.producerFactory")) .isSameAs(this.config.pf); } - @EnableBinding(TwoProcessors.class) @EnableAutoConfiguration + @Configuration public static class Config { final List outs = new ArrayList<>(); final CountDownLatch latch = new CountDownLatch(2); - @Autowired - private MessageChannel output; - - @Autowired - MessageChannel output2; - AbstractMessageListenerContainer input2Container; ProducerFactory pf; @@ -147,16 +148,19 @@ public class ConsumerProducerTransactionTests { this.latch.countDown(); } - @StreamListener(Processor.INPUT) - public void listenIn(String in) { - this.output.send(new GenericMessage<>(in.toUpperCase())); - if (in.equals("two")) { - throw new RuntimeException("fail"); - } + @Bean + public Function listenIn() { + return in -> { + if (in.equals("two")) { + throw new RuntimeException("fail"); + } + return in.toUpperCase(); + }; } - @StreamListener("input2") - public void listenIn2(String in) { + @Bean + public Function listenIn2() { + return in -> in; } @Bean @@ -187,17 +191,6 @@ public class ConsumerProducerTransactionTests { this.tm = mock; return mock; } - } - - public interface TwoProcessors extends Processor { - - @Input - SubscribableChannel input2(); - - @Output - MessageChannel output2(); - - } - } + From ba122bd39ddf86dc0d6318f031830b99b0d107b6 Mon Sep 17 00:00:00 2001 From: Oleg Zhurakousky Date: Tue, 16 Nov 2021 14:43:43 +0100 Subject: [PATCH 04/19] Changes related to GH-2245 from core --- .../cloud/stream/binder/kafka/KafkaMessageChannelBinder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java index 5334e2eff..b3211ed26 100644 --- a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java @@ -76,6 +76,7 @@ import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerPro import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; import org.springframework.cloud.stream.binder.kafka.utils.DlqDestinationResolver; import org.springframework.cloud.stream.binder.kafka.utils.DlqPartitionFunction; +import org.springframework.cloud.stream.binding.DefaultPartitioningInterceptor; import org.springframework.cloud.stream.binding.MessageConverterConfigurer.PartitioningInterceptor; import org.springframework.cloud.stream.config.ListenerContainerCustomizer; import org.springframework.cloud.stream.config.MessageSourceCustomizer; @@ -418,7 +419,7 @@ public class KafkaMessageChannelBinder extends List interceptors = ((InterceptableChannel) channel) .getInterceptors(); interceptors.forEach((interceptor) -> { - if (interceptor instanceof PartitioningInterceptor) { + if (interceptor instanceof PartitioningInterceptor || interceptor instanceof DefaultPartitioningInterceptor) { ((PartitioningInterceptor) interceptor) .setPartitionCount(partitions.size()); } From f9dfbe09f7f63be7dcea7cf7db3c9d8b50532b49 Mon Sep 17 00:00:00 2001 From: "Pommerening, Nico" Date: Wed, 17 Nov 2021 22:48:21 +0100 Subject: [PATCH 05/19] GH-1161: InteractiveQueryService improvements This PR safe guards state store instances in case there are multiple KafkaStreams instances present that have distinct application IDs but share State Store Names. Change is backwards compatible: In case no KafkaStreams association of the thread can be found, all local state stores are queried as before. In case an associated KafkaStreams Instance is found, but required StateStore is not found in this instance, a warning is issued but backwards compatibility is preserved by looking up all state stores. Store within KafkaStreams instance of thread is preferred over "foreign" store with same name. Warning is issued if requested store is not found within KafkaStreams instance of thread. The main benefit here is to get rid of randomly selecting stores across all KafkaStreams instances in case a store is contained within multiple streams instances with same name. Resolves https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1161 --- .../streams/InteractiveQueryService.java | 63 ++++++++++++++++--- ...reamsInteractiveQueryIntegrationTests.java | 8 ++- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java index 178920405..4e35c3505 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2020 the original author or authors. + * Copyright 2018-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyQueryMetadata; import org.apache.kafka.streams.StoreQueryParameters; +import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.errors.InvalidStateStoreException; import org.apache.kafka.streams.state.HostInfo; import org.apache.kafka.streams.state.QueryableStoreType; @@ -52,6 +53,7 @@ import org.springframework.util.StringUtils; * @author Soby Chacko * @author Renwei Han * @author Serhii Siryi + * @author Nico Pommerening * @since 2.1.0 */ public class InteractiveQueryService { @@ -92,15 +94,16 @@ public class InteractiveQueryService { retryTemplate.setBackOffPolicy(backOffPolicy); retryTemplate.setRetryPolicy(retryPolicy); + KafkaStreams contextSpecificKafkaStreams = getThreadContextSpecificKafkaStreams(); + return retryTemplate.execute(context -> { T store = null; - - final Set kafkaStreams = InteractiveQueryService.this.kafkaStreamsRegistry.getKafkaStreams(); - final Iterator iterator = kafkaStreams.iterator(); Throwable throwable = null; - while (iterator.hasNext()) { + if (contextSpecificKafkaStreams != null) { try { - store = iterator.next().store(StoreQueryParameters.fromNameAndType(storeName, storeType)); + store = contextSpecificKafkaStreams.store( + StoreQueryParameters.fromNameAndType( + storeName, storeType)); } catch (InvalidStateStoreException e) { // pass through.. @@ -110,10 +113,56 @@ public class InteractiveQueryService { if (store != null) { return store; } - throw new IllegalStateException("Error when retrieving state store: " + storeName, throwable); + else if (contextSpecificKafkaStreams != null) { + LOG.warn("Store " + storeName + + " could not be found in Streams context, falling back to all known Streams instances"); + } + final Set kafkaStreams = kafkaStreamsRegistry.getKafkaStreams(); + final Iterator iterator = kafkaStreams.iterator(); + while (iterator.hasNext()) { + try { + store = iterator.next() + .store(StoreQueryParameters.fromNameAndType( + storeName, storeType)); + } + catch (InvalidStateStoreException e) { + // pass through.. + throwable = e; + } + } + if (store != null) { + return store; + } + throw new IllegalStateException( + "Error when retrieving state store: " + storeName, + throwable); }); } + /** + * Retrieves the current {@link KafkaStreams} context if executing Thread is created by a Streams App (contains a matching application id in Thread's name). + * + * @return KafkaStreams instance associated with Thread + */ + private KafkaStreams getThreadContextSpecificKafkaStreams() { + return this.kafkaStreamsRegistry.getKafkaStreams().stream() + .filter(this::filterByThreadName).findAny().orElse(null); + } + + /** + * Checks if the supplied {@link KafkaStreams} instance belongs to the calling Thread by matching the Thread's name with the Streams Application Id. + * + * @param streams {@link KafkaStreams} instance to filter + * @return true if Streams Instance is associated with Thread + */ + private boolean filterByThreadName(KafkaStreams streams) { + String applicationId = kafkaStreamsRegistry.streamBuilderFactoryBean( + streams).getStreamsConfiguration() + .getProperty(StreamsConfig.APPLICATION_ID_CONFIG); + // TODO: is there some better way to find out if a Stream App created the Thread? + return Thread.currentThread().getName().contains(applicationId); + } + /** * Gets the current {@link HostInfo} that the calling kafka streams application is * running on. diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java index ab8fa087e..aff83c3a5 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2020 the original author or authors. + * Copyright 2017-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ package org.springframework.cloud.stream.binder.kafka.streams; import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.Properties; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -29,6 +30,7 @@ import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyQueryMetadata; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StoreQueryParameters; +import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.kstream.Grouped; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.Materialized; @@ -65,6 +67,7 @@ import static org.mockito.internal.verification.VerificationModeFactory.times; /** * @author Soby Chacko * @author Gary Russell + * @author Nico Pommerening */ public class KafkaStreamsInteractiveQueryIntegrationTests { @@ -102,6 +105,9 @@ public class KafkaStreamsInteractiveQueryIntegrationTests { KafkaStreamsRegistry kafkaStreamsRegistry = new KafkaStreamsRegistry(); kafkaStreamsRegistry.registerKafkaStreams(mock); Mockito.when(mock.isRunning()).thenReturn(true); + Properties mockProperties = new Properties(); + mockProperties.put(StreamsConfig.APPLICATION_ID_CONFIG, "fooApp"); + Mockito.when(mock.getStreamsConfiguration()).thenReturn(mockProperties); KafkaStreamsBinderConfigurationProperties binderConfigurationProperties = new KafkaStreamsBinderConfigurationProperties(new KafkaProperties()); binderConfigurationProperties.getStateStoreRetry().setMaxAttempts(3); From e9a8b4af7e8e2c8efc7e6d25551d0ce2aa612816 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Tue, 23 Nov 2021 15:56:52 -0500 Subject: [PATCH 06/19] GH-1170: Schema registry certificates Move classpath: resources provided as schema registry certificates into a local file system location. Adding test and docs. Resolves https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1170 --- docs/src/main/asciidoc/overview.adoc | 3 ++ .../KafkaBinderConfigurationProperties.java | 46 +++++++++++++------ ...afkaBinderConfigurationPropertiesTest.java | 20 +++++++- 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/docs/src/main/asciidoc/overview.adoc b/docs/src/main/asciidoc/overview.adoc index 317716792..096eccf64 100644 --- a/docs/src/main/asciidoc/overview.adoc +++ b/docs/src/main/asciidoc/overview.adoc @@ -151,6 +151,9 @@ Default: `false`. spring.cloud.stream.kafka.binder.certificateStoreDirectory:: When the truststore or keystore certificate location is given as a classpath URL (`classpath:...`), the binder copies the resource from the classpath location inside the JAR file to a location on the filesystem. +This is true for both broker level certificates (`ssl.truststore.location` and `ssl.keystore.location`) and certificates intended for schema registry (`schema.registry.ssl.truststore.location` and `schema.registry.ssl.keystore.location`). +Keep in mind that the truststore and keystore classpath locations must be provided under `spring.cloud.stream.kafka.binder.configuration...`. +For example, `spring.cloud.stream.kafka.binder.configuration.ssl.truststore.location`, ``spring.cloud.stream.kafka.binder.configuration.schema.registry.ssl.truststore.location`, etc. The file will be moved to the location specified as the value for this property which must be an existing directory on the filesystem that is writable by the process running the application. If this value is not set and the certificate file is a classpath resource, then it will be moved to System's temp directory as returned by `System.getProperty("java.io.tmpdir")`. This is also true, if this value is present, but the directory cannot be found on the filesystem or is not writable. diff --git a/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java b/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java index 06506bf44..c53566eef 100644 --- a/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java +++ b/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 the original author or authors. + * Copyright 2015-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -163,24 +163,44 @@ public class KafkaBinderConfigurationProperties { private void moveCertsToFileSystemIfNecessary() { try { - final String trustStoreLocation = this.configuration.get("ssl.truststore.location"); - if (trustStoreLocation != null && trustStoreLocation.startsWith("classpath:")) { - final String fileSystemLocation = moveCertToFileSystem(trustStoreLocation, this.certificateStoreDirectory); - // Overriding the value with absolute filesystem path. - this.configuration.put("ssl.truststore.location", fileSystemLocation); - } - final String keyStoreLocation = this.configuration.get("ssl.keystore.location"); - if (keyStoreLocation != null && keyStoreLocation.startsWith("classpath:")) { - final String fileSystemLocation = moveCertToFileSystem(keyStoreLocation, this.certificateStoreDirectory); - // Overriding the value with absolute filesystem path. - this.configuration.put("ssl.keystore.location", fileSystemLocation); - } + moveBrokerCertsIfApplicable(); + moveSchemaRegistryCertsIfApplicable(); } catch (Exception e) { throw new IllegalStateException(e); } } + private void moveBrokerCertsIfApplicable() throws IOException { + final String trustStoreLocation = this.configuration.get("ssl.truststore.location"); + if (trustStoreLocation != null && trustStoreLocation.startsWith("classpath:")) { + final String fileSystemLocation = moveCertToFileSystem(trustStoreLocation, this.certificateStoreDirectory); + // Overriding the value with absolute filesystem path. + this.configuration.put("ssl.truststore.location", fileSystemLocation); + } + final String keyStoreLocation = this.configuration.get("ssl.keystore.location"); + if (keyStoreLocation != null && keyStoreLocation.startsWith("classpath:")) { + final String fileSystemLocation = moveCertToFileSystem(keyStoreLocation, this.certificateStoreDirectory); + // Overriding the value with absolute filesystem path. + this.configuration.put("ssl.keystore.location", fileSystemLocation); + } + } + + private void moveSchemaRegistryCertsIfApplicable() throws IOException { + String trustStoreLocation = this.configuration.get("schema.registry.ssl.truststore.location"); + if (trustStoreLocation != null && trustStoreLocation.startsWith("classpath:")) { + final String fileSystemLocation = moveCertToFileSystem(trustStoreLocation, this.certificateStoreDirectory); + // Overriding the value with absolute filesystem path. + this.configuration.put("schema.registry.ssl.truststore.location", fileSystemLocation); + } + final String keyStoreLocation = this.configuration.get("schema.registry.ssl.keystore.location"); + if (keyStoreLocation != null && keyStoreLocation.startsWith("classpath:")) { + final String fileSystemLocation = moveCertToFileSystem(keyStoreLocation, this.certificateStoreDirectory); + // Overriding the value with absolute filesystem path. + this.configuration.put("schema.registry.ssl.keystore.location", fileSystemLocation); + } + } + private String moveCertToFileSystem(String classpathLocation, String fileSystemLocation) throws IOException { File targetFile; final String tempDir = System.getProperty("java.io.tmpdir"); diff --git a/spring-cloud-stream-binder-kafka-core/src/test/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationPropertiesTest.java b/spring-cloud-stream-binder-kafka-core/src/test/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationPropertiesTest.java index 776bc4b9b..48df4962a 100644 --- a/spring-cloud-stream-binder-kafka-core/src/test/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationPropertiesTest.java +++ b/spring-cloud-stream-binder-kafka-core/src/test/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationPropertiesTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 the original author or authors. + * Copyright 2018-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -142,4 +142,22 @@ public class KafkaBinderConfigurationPropertiesTest { assertThat(configuration.get("ssl.keystore.location")).isEqualTo( Paths.get(Files.currentFolder().toString(), "target", "testclient.keystore").toString()); } + + @Test + public void testCertificateFilesAreMovedForSchemaRegistryConfiguration() { + KafkaProperties kafkaProperties = new KafkaProperties(); + KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties = + new KafkaBinderConfigurationProperties(kafkaProperties); + final Map configuration = kafkaBinderConfigurationProperties.getConfiguration(); + configuration.put("schema.registry.ssl.truststore.location", "classpath:testclient.truststore"); + configuration.put("schema.registry.ssl.keystore.location", "classpath:testclient.keystore"); + kafkaBinderConfigurationProperties.setCertificateStoreDirectory("target"); + + kafkaBinderConfigurationProperties.getKafkaConnectionString(); + + assertThat(configuration.get("schema.registry.ssl.truststore.location")).isEqualTo( + Paths.get(Files.currentFolder().toString(), "target", "testclient.truststore").toString()); + assertThat(configuration.get("schema.registry.ssl.keystore.location")).isEqualTo( + Paths.get(Files.currentFolder().toString(), "target", "testclient.keystore").toString()); + } } From e7bf404fce3c2a99ebe11aadd1d0dd3b60f6fa16 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Wed, 24 Nov 2021 13:12:52 -0500 Subject: [PATCH 07/19] Fix PartitioningInterceptor CCE The newly added DefaultPartitioningInteceptor must be explicitly checked in order to avoid a CCE. Related to resolving https://github.com/spring-cloud/spring-cloud-stream/issues/2245 Specifically for this: https://github.com/spring-cloud/spring-cloud-stream/issues/2245#issuecomment-977663452 --- .../stream/binder/kafka/KafkaMessageChannelBinder.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java index b3211ed26..6a4973def 100644 --- a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java @@ -419,10 +419,14 @@ public class KafkaMessageChannelBinder extends List interceptors = ((InterceptableChannel) channel) .getInterceptors(); interceptors.forEach((interceptor) -> { - if (interceptor instanceof PartitioningInterceptor || interceptor instanceof DefaultPartitioningInterceptor) { + if (interceptor instanceof PartitioningInterceptor) { ((PartitioningInterceptor) interceptor) .setPartitionCount(partitions.size()); } + else if (interceptor instanceof DefaultPartitioningInterceptor) { + ((DefaultPartitioningInterceptor) interceptor) + .setPartitionCount(partitions.size()); + } }); } From 0be87c366615d78c4ebb6807993ee17b3f961ee5 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Mon, 29 Nov 2021 12:03:52 -0500 Subject: [PATCH 08/19] New tips-tricks-recipes section in docs Migrate the recipe section in Spring Cloud Stream Samples repository as Tips, Tricks and Receipes in Kafka binder main docs. Resolves https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1173 Resolves #1174 --- .../spring-cloud-stream-binder-kafka.adoc | 2 + docs/src/main/asciidoc/tips.adoc | 865 ++++++++++++++++++ 2 files changed, 867 insertions(+) create mode 100644 docs/src/main/asciidoc/tips.adoc diff --git a/docs/src/main/asciidoc/spring-cloud-stream-binder-kafka.adoc b/docs/src/main/asciidoc/spring-cloud-stream-binder-kafka.adoc index fcbc7b469..79d871825 100644 --- a/docs/src/main/asciidoc/spring-cloud-stream-binder-kafka.adoc +++ b/docs/src/main/asciidoc/spring-cloud-stream-binder-kafka.adoc @@ -44,6 +44,8 @@ include::partitions.adoc[] include::kafka-streams.adoc[] +include::tips.adoc[] + = Appendices [appendix] include::building.adoc[] diff --git a/docs/src/main/asciidoc/tips.adoc b/docs/src/main/asciidoc/tips.adoc new file mode 100644 index 000000000..ee8f954e2 --- /dev/null +++ b/docs/src/main/asciidoc/tips.adoc @@ -0,0 +1,865 @@ +== Tips, Tricks and Recipes + +=== Simple DLQ with Kafka + +==== Problem Statement + +As a developer, I want to write a consumer application that processes records from a Kafka topic. +However, if some error occurs in processing, I don't want the application to stop completely. +Instead, I want to send the record in error to a DLT (Dead-Letter-Topic) and then continue processing new records. + +==== Solution + +The solution for this problem is to use the DLQ feature in Spring Cloud Stream. +For the purposes of this discussion, let us assume that the following is our processor function. + +``` +@Bean +public Consumer processData() { + return s -> { + throw new RuntimeException(); + }; +``` + +This is a very trivial function that throws an exception for all the records that it processes, but you can take this function and extend it to any other similar situations. + +In order to send the records in error to a DLT, we need to provide the following configuration. + +``` +spring.cloud.stream: + bindings: + processData-in-0: + group: my-group + destination: input-topic + kafka: + bindings: + processData-in-0: + consumer: + enableDlq: true + dlqName: input-topic-dlq +``` + +In order to activate DLQ, the application must provide a group name. +Anonymous consumers cannot use the DLQ facilities. +We also need to enable DLQ by setting the `enableDLQ` property on the Kafka consumer binding to `true`. +Finally, we can optionally provide the DLT name by providing the `dlqName` on Kafka consumer binding, which otherwise default to `input-topic-dlq.my-group.error` in this case. + +Note that in the example consumer provided above, the type of the payload is `byte[]`. +By default, the DLQ producer in Kafka binder expects the payload of type `byte[]`. +If that is not the case, then we need to provide the configuration for proper serializer. +For example, let us re-write the consumer function as below: + +``` +@Bean +public Consumer processData() { + return s -> { + throw new RuntimeException(); + }; +} +``` + +Now, we need to tell Spring Cloud Stream, how we want to serialize the data when writing to the DLT. +Here is the modified configuration for this scenario: + +``` +spring.cloud.stream: + bindings: + processData-in-0: + group: my-group + destination: input-topic + kafka: + bindings: + processData-in-0: + consumer: + enableDlq: true + dlqName: input-topic-dlq + dlqProducerProperties: + configuration: + value.serializer: org.apache.kafka.common.serialization.StringSerializer + +``` + +=== DLQ with Advanced Retry Options + +==== Problem Statement + +This is similar to the recipe above, but as a developer I would like to configure the way retries are handled. + +==== Solution + +If you followed the above recipe, then you get the default retry options built into the Kafka binder when the processing encounters an error. + +By default, the binder retires for a maximum of 3 attempts with a one second initial delay, 2.0 multiplier with each back off with a max delay of 10 seconds. +You can change all these configurations as below: + +``` +spring.cloud.stream.bindings.processData-in-0.consumer.maxAtttempts +spring.cloud.stream.bindings.processData-in-0.consumer.backOffInitialInterval +spring.cloud.stream.bindings.processData-in-0.consumer.backOffMultipler +spring.cloud.stream.bindings.processData-in-0.consumer.backOffMaxInterval +``` + +If you want, you can also provide a list of retryable exceptions by providing a map of boolean values. +For example, + +``` +spring.cloud.stream.bindings.processData-in-0.consumer.retryableExceptions.java.lang.IllegalStateException=true +spring.cloud.stream.bindings.processData-in-0.consumer.retryableExceptions.java.lang.IllegalArgumentException=false +``` + +By default, any exceptions not listed in the map above will be retried. +If that is not desired, then you can disable that by providing, + +``` +spring.cloud.stream.bindings.processData-in-0.consumer.defaultRetryable=false +``` + +You can also provide your own `RetryTemplate` and mark it as `@StreamRetryTemplate` which will be scanned and used by the binder. +This is useful when you want more sophisticated retry strategies and policies. + +If you have multiple `@StreamRetryTemplate` beans, then you can specify which one your binding wants by using the property, + +``` +spring.cloud.stream.bindings.processData-in-0.consumer.retry-template-name= +``` + +=== Handling Deserialization errors with DLQ + +==== Problem Statement + +I have a processor that encounters a deserilzartion exception in Kafka consumer. +I would expect that the Spring Cloud Stream DLQ mechanism will catch that scenario, but it does not. +How can I handle this? + +==== Solution + +The normal DLQ mechanism offered by Spring Cloud Stream will not help when Kafka consumer throws an irrecoverable deserialization excepion. +This is because, this exception happens even before the consumer's `poll()` method returns. +Spring for Apache Kafka project offers some great ways to help the binder with this situation. +Let us explore those. + +Assuming this is our function: + +``` +@Bean +public Consumer functionName() { + return s -> { + System.out.println(s); + }; +} +``` + +It is a trivial function that takes a `String` parameter. + +We want to bypass the message converters provided by Spring Cloud Stream and want to use native deserializers instead. +In the case of `String` types, it does not make much sense, but for more complex types like AVRO etc. you have to rely on external deserializers and therefore want to delegate the conversion to Kafka. + +Now when the consumer receives the data, let us assume that there is a bad record that causes a deserilziation errror, maybe someone passed an `Integer` instead of a `String` for example. +In that case, if you don't do something in the application, the excption will be propagated through the chain and your application will exit eventually. + +In order to handle this, you can add a `ListenerContainerCustomizer` `@Bean` that configures a `SeekToCurrentErrorHandler`. +This `SeekToCurrentErrorHandler` is configured with a `DeadLetterPublishingRecoverer`. +We also need to configure an `ErrorHandlingDeserializer` for the consumer. +That sounds like a lot of complex things, but in reality, it boils down to these 3 beans in this case. + +``` +@Bean + public ListenerContainerCustomizer> customizer(SeekToCurrentErrorHandler errorHandler) { + return (container, dest, group) -> { + container.setErrorHandler(errorHandler); + }; + } +``` + +``` + @Bean + public SeekToCurrentErrorHandler errorHandler(DeadLetterPublishingRecoverer deadLetterPublishingRecoverer) { + return new SeekToCurrentErrorHandler(deadLetterPublishingRecoverer); + } +``` + +``` + @Bean + public DeadLetterPublishingRecoverer publisher(KafkaOperations bytesTemplate) { + return new DeadLetterPublishingRecoverer(bytesTemplate); + } +``` + +Let us analyze each of them. +The first one is the `ListenerContainerCustomizer` bean that takes a `SeekToCurrentErrorHandler`. +The container is now customized with that particular error handler. +You can learn more about container customization https://docs.spring.io/spring-cloud-stream/docs/current/reference/html/spring-cloud-stream.html#_advanced_consumer_configuration[here]. + +The second bean is the `SeekToCurrentErrorHandler` that is configured with a publishing to a `DLT`. +See https://docs.spring.io/spring-kafka/docs/current/reference/html/#seek-to-current[here] for more details on `SeekToCurrentErrorHandler`. + +The third bean is the `DeadLetterPublishingRecoverer` that is ultimately responsible for sending to the `DLT`. +By default, the `DLT` topic is named as the ORIGINAL_TOPIC_NAME.DLT. +You can change that though. +See the https://docs.spring.io/spring-kafka/docs/current/reference/html/#dead-letters[docs] for more details. + + +We also need to configure an https://docs.spring.io/spring-kafka/docs/current/reference/html/#error-handling-deserializer[ErrorHandlingDeserializer] through application config. + +The `ErrorHandlingDeserializer` delegates to the actual deserializer. +In case of errors, it sets key/value of the record to be null and includes the raw bytes of the message. +It then sets the exception in a header and passes this record to the listener, which then calls the registered error handler. + +Following is the configuration required: + +``` +spring.cloud.stream: + function: + definition: functionName + bindings: + functionName-in-0: + group: group-name + destination: input-topic + consumer: + use-native-decoding: true + kafka: + bindings: + functionName-in-0: + consumer: + enableDlq: true + dlqName: dlq-topic + dlqProducerProperties: + configuration: + value.serializer: org.apache.kafka.common.serialization.StringSerializer + configuration: + value.deserializer: org.springframework.kafka.support.serializer.ErrorHandlingDeserializer + spring.deserializer.value.delegate.class: org.apache.kafka.common.serialization.StringDeserializer +``` + +We are providing the `ErrorHandlingDeserializer` through the `configuration` property on the binding. +We are also indicating that the actual deserializer to delegate is the `StringDeserializer`. + +Keep in mind that none of the dlq properties above are relevant for the discussions in this recipe. +They are purely meant for addressing any application level errors only. + +=== Basic offset management in Kafka binder + +==== Problem Statement + +I want to write a Spring Cloud Stream Kafka consumer applicaiton and not sure about how it manages Kafka consumer offsets. +Can you exaplain? + +==== Solution + +We encourage you read the https://docs.spring.io/spring-cloud-stream-binder-kafka/docs/current/reference/html/spring-cloud-stream-binder-kafka.html#reset-offsets[docs] section on this to get a thorough understanding on it. + +Here is it in a gist: + +Kafka supports two types of offsets to start with by default - `earliest` and `latest`. +Their semantics are self-explanatory from their names. + +Assuming you are running the consumer for the first time. +If you miss the group.id in your Spring Cloud Stream application, then it becomes an anonymous consumer. +Whenever, you have an anonymous consumer, in that case, Spring Cloud Stream application by default will start from the `latest` available offset in the topic partition. +On the other hand, if you explicitly specify a group.id, then by default, the Spring Cloud Stream application will start from the `earliest` available offset in the topic partiton. + +In both cases above (consumers with explicit groups and anonymous groups), the starting offset can be switched around by using the property `spring.cloud.stream.kafka.bindings..consumer.startOffset` and setting it to either `earliest` or `latest`. + +Now, assume that you already ran the consumer before and now starting it again. +In this case, the starting offset semantics in the above case do not apply as the consumer finds an already committed offset for the consumer group (In the case of an anonymous consumer, although the application does not provide a group.id, the binder will auto generate one for you). +It simply picks up from the last committed offset onward. +This is true, even when you have a `startOffset` value provided. + +However, you can override the default behavior where the consumer starts from the last committed offset by using the `resetOffsets` property. +In order to do that, set the property `spring.cloud.stream.kafka.bindings..consumer.resetOffsets` to `true` (which is `false` by default). +Then make sure you provide the `startOffset` value (either `earliest` or `latest`). +When you do that and then start the consumer application, each time you start, it starts as if this is starting for the first time and ignore any committed offsets for the partition. + +=== Seeking to arbitrary offsets in Kafka + +==== Problem Statement + +Using Kafka binder, I know that it can set the offset to either `earliest` or `latest`, but I have a requirement to seek the offset to something in the middle, an arbitrary offset. +Is there a way to achieve this using Spring Cloud Stream Kafka biner? + +==== Solution + +Previously we saw how Kafka binder allows you to tackle basic offset management. +By default, the binder does not allow you to rewind to an arbitrary offset, at least through the mechanism we saw in that reipce. +However, there are some low-level strategies that the binder provides to achieve this use case. +Let's explore them. + +First of all, when you want to reset to an arbitrary offset other than `earliest` or `latest`, make sure to leave the `resetOffsets` configuration to its defaults, which is `false`. +Then you have to provide a custom bean of type `KafkaBindingRebalanceListener`, which will be injected into all consumer bindings. +It is an interface that comes with a few default methods, but here is the method that we are interested in: + +``` +/** + * Invoked when partitions are initially assigned or after a rebalance. Applications + * might only want to perform seek operations on an initial assignment. While the + * 'initial' argument is true for each thread (when concurrency is greater than 1), + * implementations should keep track of exactly which partitions have been sought. + * There is a race in that a rebalance could occur during startup and so a topic/ + * partition that has been sought on one thread may be re-assigned to another + * thread and you may not wish to re-seek it at that time. + * @param bindingName the name of the binding. + * @param consumer the consumer. + * @param partitions the partitions. + * @param initial true if this is the initial assignment on the current thread. + */ + default void onPartitionsAssigned(String bindingName, Consumer consumer, + Collection partitions, boolean initial) { + // do nothing + } +``` + +Let us look at the details. + +In essence, this method will be invoked each time during the initial assignment for a topic partition or after a rebalance. +For better illustration, let us assume that our topic is `foo` and it has 4 partitions. +Initially, we are only starting a single consumer in the group and this consumer will consume from all partitions. +When the consumer starts for the first time, all 4 partitions are getting initially assigned. +However, we do not want to start the partitions to consume at the defaults (`earliest` since we define a group), rather for each partition, we want them to consume after seeking to arbitrary offsets. +Imagine that you have a business case to consume from certain offsets as below. + +``` +Partition start offset + +0 1000 +1 2000 +2 2000 +3 1000 +``` + +This could be achieved by implementing the above method as below. + +``` + +@Override +public void onPartitionsAssigned(String bindingName, Consumer consumer, Collection partitions, boolean initial) { + + Map topicPartitionOffset = new HashMap<>(); + topicPartitionOffset.put(new TopicPartition("foo", 0), 1000L); + topicPartitionOffset.put(new TopicPartition("foo", 1), 2000L); + topicPartitionOffset.put(new TopicPartition("foo", 2), 2000L); + topicPartitionOffset.put(new TopicPartition("foo", 3), 1000L); + + if (initial) { + partitions.forEach(tp -> { + if (topicPartitionOffset.containsKey(tp)) { + final Long offset = topicPartitionOffset.get(tp); + try { + consumer.seek(tp, offset); + } + catch (Exception e) { + // Handle excpetions carefully. + } + } + }); + } +} +``` + +This is just a rudimentary implementation. +Real world use cases are much more complex than this and you need to adjust accordingly, but this certainly gives you a basic sketch. +When consumer `seek` fails, it may throw some runtime exceptions and you need to decide what to do in those cases. + +==== What if we start a second consumer with the same group id? + +When we add a second consumer, a rebalance will occur and some partitions will be moved around. +Let's say that the new consumer gets partitions `2` and `3`. +When this new Spring Cloud Stream consumer calls this `onPartitionsAssigned` method, it will see that this is the initial assignment for partititon `2` and `3` on this consumer. +Therefore, it will do the seek operation becuase of the conditional check on the `initial` argument. +In the case of the first consumer, it now only has partitons `0` and `1` +However, for this consumer it was simply a rebalance event and not considered as an intial assignment. +Thus, it will not re-seek to the given offsets because of the conditional check on the `initial` argument. + +=== How do I manually acknowledge using Kafka binder? + +==== Problem Statement + +Using Kafka binder, I want to manually acknowledge messages in my consumer. +How do I do that? + +==== Solution + +By default, Kafka binder delegates to the default commit settings in Spring for Apache Kafka project. +The default `ackMode` in Spring Kafka is `batch`. +See https://docs.spring.io/spring-kafka/docs/current/reference/html/#committing-offsets[here] for more details on that. + +There are situations in which you want to disable this default commit behavior and rely on manual commits. +Following steps allow you to do that. + +Set the property `spring.cloud.stream.kafka.bindings..consumer.ackMode` to either `MANUAL` or `MANUAL_IMMEDIATE`. +When it is set like that, then there will be a header called `kafka_acknowledgment` (from `KafkaHeaders.ACKNOWLEDGMENT`) present in the message received by the consumer method. + +For example, imagine this as your consumer method. + +``` +@Bean +public Consumer> myConsumer() { + return msg -> { + Acknowledgment acknowledgment = message.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT, Acknowledgment.class); + if (acknowledgment != null) { + System.out.println("Acknowledgment provided"); + acknowledgment.acknowledge(); + } + }; +} +``` + +Then you set the property `spring.cloud.stream.bindings.myConsumer-in-0.consumer.ackMode` to `MANUAL` or `MANUAL_IMMEDIATE`. + +=== How do I override the default binding names in Spring Cloud Stream? + +==== Problem Statement + +Spring Cloud Stream creates default bindings based on the function definition and signature, but how do I override these to more domain friendly names? + +==== Solution + +Assume that following is your function signature. + +``` +@Bean +public Function uppercase(){ +... +} +``` + +By default, Spring Cloud Stream will create the bindings as below. + +1. uppercase-in-0 +2. uppercase-out-0 + +You can override these bindings to something by using the following properties. + +``` +spring.cloud.stream.function.bindings.uppercase-in-0=my-transformer-in +spring.cloud.stream.function.bindings.uppercase-out-0=my-transformer-out +``` + +After this, all binding properties must be made on the new names, `my-transformer-in` and `my-transformer-out`. + +Here is another example with Kafka Streams and multiple inputs. + +``` +@Bean +public BiFunction, KTable, KStream> processOrder() { +... +} +``` + +By default, Spring Cloud Stream will create three different binding names for this function. + +1. processOrder-in-0 +2. processOrder-in-1 +3. processOrder-out-0 + +You have to use these binding names each time you want to set some configuration on these bindings. +You don't like that, and you want to use more domain-friendly and readable binding names, for example, something like. + +1. orders +2. accounts +3. enrichedOrders + +You can easily do that by simply setting these three properties + +1. spring.cloud.stream.function.bindings.processOrder-in-0=orders +2. spring.cloud.stream.function.bindings.processOrder-in-1=accounts +3. spring.cloud.stream.function.bindings.processOrder-out-0=enrichedOrders + +Once you do that, it overrides the default binding names and any properties that you want to set on them must be on these new binding names. + +=== How do I send a message key as part of my record? + +==== Problem Statement + +I need to send a key along with the payload of the record, is there a way to do that in Spring Cloud Stream? + +==== Solution + +It is often necessary that you want to send associative data structure like a map as the record with a key and value. +Spring Cloud Stream allows you to do that in a straightforward manner. +Following is a basic blueprint for doing this, but you may want to adapt it to your paricular use case. + +Here is sample producer method (aka `Supplier`). + +``` +@Bean +public Supplier> supplier() { + return () -> MessageBuilder.withPayload("foo").setHeader(KafkaHeaders.MESSAGE_KEY, "my-foo").build(); +} +``` + +This is a trivial function that sends a message with a `String` payload, but also with a key. +Note that we set the key as a message header using `KafkaHeaders.MESSAGE_KEY`. + +If you want to change the key from the default `kafka_messageKey`, then in the configuration, we need to specify this property: + +``` +spring.cloud.stream.kafka.bindings.supplier-out-0.producer.messageKeyExpression=headers['my-special-key'] +``` + +Please note that we use the binding name `supplier-out-0` since that is our function name, please update accordingly. + +Then, we use this new key when we produce the message. + +=== How do I use native serializer and deserializer instead of message conversion done by Spring Cloud Stream? + +==== Problem Statement + +Instead of using the message converters in Spring Cloud Stream, I want to use native Serializer and Deserializer in Kafka. +By default, Spring Cloud Stream takes care of this conversion using its internal built-in message converters. +How can I bypass this and delegate the responsibility to Kafka? + +==== Solution + +This is really easy to do. + +All you have to do is to provide the following property to enable native serialization. + +``` +spring.cloud.stream.kafka.bindings..producer.useNativeEncoding: true +``` + +Then, you need to also set the serailzers. +There are a couple of ways to do this. + +``` +spring.cloud.stream.kafka.bindings..producer.configurarion.key.serializer: org.apache.kafka.common.serialization.StringSerializer +spring.cloud.stream.kafka.bindings..producer.configurarion.value.serializer: org.apache.kafka.common.serialization.StringSerializer +``` + +or using the binder configuration. + +``` +spring.cloud.stream.kafka.binder.configurarion.key.serializer: org.apache.kafka.common.serialization.StringSerializer +spring.cloud.stream.kafka.binder.configurarion.value.serializer: org.apache.kafka.common.serialization.StringSerializer +``` + +When using the binder way, it is applied against all the bindings whereas setting them at the bindings are per binding. + +On the deserializing side, you just need to provide the deserializers as configuration. + +For example, + +``` +spring.cloud.stream.kafka.bindings..consumer.configurarion.key.deserializer: org.apache.kafka.common.serialization.StringDeserializer +spring.cloud.stream.kafka.bindings..producer.configurarion.value.deserializer: org.apache.kafka.common.serialization.StringDeserializer +``` + +You can also set them at the binder level. + +There is an optional property that you can set to force native decoding. + +``` +spring.cloud.stream.kafka.bindings..consumer.useNativeDecoding: true +``` + +However, in the case of Kafka binder, this is unncessary, as by the time it reaches the binder, Kafka already deserializes them using the configured deserializers. + +=== Explain how offset resetting work in Kafka Streams binder + +==== Problem Statement + +By default, Kafka Streams binder always starts from the earliest offset for a new consumer. +Sometimes, it is beneficial or required by the application to start from the latest offset. +Kafka Streams binder allows you to do that. + +==== Solution + +Before we look at the solution, let us look at the following scenario. + +``` +@Bean +public BiConsumer, KTable> myBiConsumer{ + (s, t) -> s.join(t, ...) + ... +} +``` + +We have a `BiConsumer` bean that requires two input bindings. +In this case, the first binding is for a `KStream` and the second one is for a `KTable`. +When running this application for the first time, by default, both bindings start from the `earliest` offset. +What about I want to start from the `latest` offset due to some requirements? +You can do this by enabling the following properties. + +``` +spring.cloud.stream.kafka.streams.bindings.myBiConsumer-in-0.consumer.startOffset: latest +spring.cloud.stream.kafka.streams.bindings.myBiConsumer-in-1.consumer.startOffset: latest +``` + +If you want only one binding to start from the `latest` offset and the other to consumer from the default `earliest`, then leave the latter binding out from the configuration. + +Keep in mind that, once there are committed offsets, these setting are *not* honored and the committed offsets take precedence. + +=== Keeping track of successful sending of records (producing) in Kafka + +==== Problem Statement + +I have a Kafka producer application and I want to keep track of all my successful sedings. + +==== Solution + +Let us assume that we have this following supplier in the application. + +``` +@Bean + public Supplier> supplier() { + return () -> MessageBuilder.withPayload("foo").setHeader(KafkaHeaders.MESSAGE_KEY, "my-foo").build(); + } +``` + +Then, we need to define a new `MessageChannel` bean to capture all the successful send information. + +``` +@Bean + public MessageChannel fooRecordChannel() { + return new DirectChannel(); + } +``` + +Next, define this property in the application configuration to provide the bean name for the `recordMetadataChannel`. + +``` +spring.cloud.stream.kafka.bindings.supplier-out-0.producer.recordMetadataChannel: fooRecordChannel +``` + +At this point, successful sent information will be sent to the `fooRecordChannel`. + +You can write an `IntegrationFlow` as below to see the information. + +``` +@Bean +public IntegrationFlow integrationFlow() { + return f -> f.channel("fooRecordChannel") + .handle((payload, messageHeaders) -> payload); +} +``` + +In the `handle` method, the payload is what got sent to Kafka and the message headers contain a special key called `kafka_recordMetadata`. +Its value is a `RecordMetadata` that contains information about topic partition, current offset etc. + +=== Adding custom header mapper in Kafka + +==== Problem Statement + +I have a Kafka producer application that sets some headers, but they are missing in the consumer application. Why is that? + +==== Solution + +Under normal circumstances, this should be fine. + +Imagine, you have the following producer. + +``` +@Bean +public Supplier> supply() { + return () -> MessageBuilder.withPayload("foo").setHeader("foo", "bar").build(); +} +``` + +On the consumer side, you should still see the header "foo", and the following should not give you any issues. + +``` +@Bean +public Consumer> consume() { + return s -> { + final String foo = (String)s.getHeaders().get("foo"); + System.out.println(foo); + }; +} +``` + +If you provide a https://docs.spring.io/spring-cloud-stream-binder-kafka/docs/3.1.3/reference/html/spring-cloud-stream-binder-kafka.html#_kafka_binder_properties[custom header mapper] in the application, then this won't work. +Let's say you have an empty `KafkaHeaderMapper` in the application. + +``` +@Bean +public KafkaHeaderMapper kafkaBinderHeaderMapper() { + return new KafkaHeaderMapper() { + @Override + public void fromHeaders(MessageHeaders headers, Headers target) { + + } + + @Override + public void toHeaders(Headers source, Map target) { + + } + }; +} +``` + +If that is your implementation, then you will miss the `foo` header on the consumer. +Chances are that, you may have some logic inside those `KafkaHeaderMapper` methods. +You need the following to populate the `foo` header. + +``` +@Bean +public KafkaHeaderMapper kafkaBinderHeaderMapper() { + return new KafkaHeaderMapper() { + @Override + public void fromHeaders(MessageHeaders headers, Headers target) { + final String foo = (String) headers.get("foo"); + target.add("foo", foo.getBytes()); + } + + @Override + public void toHeaders(Headers source, Map target) { + final Header foo = source.lastHeader("foo"); + target.put("foo", new String(foo.value())); + } + } +``` + +That will properly populate the `foo` header from the producer to consumer. + +==== Special note on the id header + +In Spring Cloud Stream, the `id` header is a special header, but some applications may want to have special custom id headers - something like `custom-id` or `ID` or `Id`. +The first one (`custom-id`) will propagate without any custom header mapper from producer to consumer. +However, if you produce with a variant of the framework reserved `id` header - such as `ID`, `Id`, `iD` etc. then you will run into issues with the internals of the framework. +See this https://stackoverflow.com/questions/68412600/change-the-behaviour-in-spring-cloud-stream-make-header-matcher-case-sensitive[StackOverflow thread] fore more context on this use case. +In that case, you must use a custom `KafkaHeaderMapper` to map the case-sensitive id header. +For example, let's say you have the following producer. + +``` +@Bean +public Supplier> supply() { + return () -> MessageBuilder.withPayload("foo").setHeader("Id", "my-id").build(); +} +``` + +The header `Id` above will be gone from the consuming side as it clashes with the framework `id` header. +You can provide a custom `KafkaHeaderMapper` to solve this issue. + +``` +@Bean +public KafkaHeaderMapper kafkaBinderHeaderMapper1() { + return new KafkaHeaderMapper() { + @Override + public void fromHeaders(MessageHeaders headers, Headers target) { + final String myId = (String) headers.get("Id"); + target.add("Id", myId.getBytes()); + } + + @Override + public void toHeaders(Headers source, Map target) { + final Header Id = source.lastHeader("Id"); + target.put("Id", new String(Id.value())); + } + }; +} +``` + +By doing this, both `id` and `Id` headers will be available from the producer to the consumer side. + +=== Producing to multiple topics in transaction + +==== Problem Statement + +How do I produce transactional messages to multiple Kafka topics? + +For more context, see this https://stackoverflow.com/questions/68928091/dlq-bounded-retry-and-eos-when-producing-to-multiple-topics-using-spring-cloud[StackOverflow question]. + +==== Solution + +Use transactional support in Kafka binder for transactions and then provide an `AfterRollbackProcessor`. +In order to produce to multiple topics, use `StreamBridge` API. + +Below are the code snippets for this: + +``` +@Autowired +StreamBridge bridge; + +@Bean +Consumer input() { + return str -> { + System.out.println(str); + this.bridge.send("left", str.toUpperCase()); + this.bridge.send("right", str.toLowerCase()); + if (str.equals("Fail")) { + throw new RuntimeException("test"); + } + }; +} + +@Bean +ListenerContainerCustomizer> customizer(BinderFactory binders) { + return (container, dest, group) -> { + ProducerFactory pf = ((KafkaMessageChannelBinder) binders.getBinder(null, + MessageChannel.class)).getTransactionalProducerFactory(); + KafkaTemplate template = new KafkaTemplate<>(pf); + DefaultAfterRollbackProcessor rollbackProcessor = rollbackProcessor(template); + container.setAfterRollbackProcessor(rollbackProcessor); + }; +} + +DefaultAfterRollbackProcessor rollbackProcessor(KafkaTemplate template) { + return new DefaultAfterRollbackProcessor<>( + new DeadLetterPublishingRecoverer(template), new FixedBackOff(2000L, 2L), template, true); +} + +``` + +==== Required Configuration + +``` +spring.cloud.stream.kafka.binder.transaction.transaction-id-prefix: tx- +spring.cloud.stream.kafka.binder.required-acks=all +spring.cloud.stream.bindings.input-in-0.group=foo +spring.cloud.stream.bindings.input-in-0.destination=input +spring.cloud.stream.bindings.left.destination=left +spring.cloud.stream.bindings.right.destination=right + +spring.cloud.stream.kafka.bindings.input-in-0.consumer.maxAttempts=1 +``` + +in order to test, you can use the following: + +``` +@Bean +public ApplicationRunner runner(KafkaTemplate template) { + return args -> { + System.in.read(); + template.send("input", "Fail".getBytes()); + template.send("input", "Good".getBytes()); + }; +} +``` + +Some important notes: + +Please ensure that you don't have any DLQ settings on the application configuration as we manually configure DLT (By default it will be published to a topic named `input.DLT` based on the initial consumer function). +Also, reset the `maxAttempts` on consumer binding to `1` in order to avoid retries by the binder. +It will be max tried a total of 3 in the example above (initial try + the 2 attempts in the `FixedBackoff`). + +See the https://stackoverflow.com/questions/68928091/dlq-bounded-retry-and-eos-when-producing-to-multiple-topics-using-spring-cloud[StackOverflow thread] for more details on how to test this code. +If you are using Spring Cloud Stream to test it by adding more consumer functions, make sure to set the `isolation-level` on the consumer binding to `read-committed`. + +This https://stackoverflow.com/questions/68941306/spring-cloud-stream-database-transaction-does-not-roll-back[StackOverflow thread] is also related to this discussion. + +=== Pitfalls to avoid when running multiple pollable consumers + +==== Problem Statement + +How can I run multiple instances of the pollable consumers and generate unique `client.id` for each instance? + +==== Solution + +Assuming that I have the following definition: + +``` +spring.cloud.stream.pollable-source: foo +spring.cloud.stream.bindings.foo-in-0.group: my-group +``` + +When running the application, the Kafka consumer generates a client.id (something like `consumer-my-group-1`). +For each instance of the application that is running, this `client.id` will be the same, causing unexpected issues. + +In order to fix this, you can add the following property on each instance of the application: + +``` +spring.cloud.stream.kafka.bindings.foo-in-0.consumer.configuration.client.id=${client.id} +``` + +See this https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1139[GitHub issue] for more details. + From be474f643a5a3f3429670c4c513867ca18d694c3 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Mon, 29 Nov 2021 20:12:45 -0500 Subject: [PATCH 09/19] KafkaStreams binder health check improvements Allow health checks on KafkaStreams processors that are currently stopped through actuator bindings endpoint. Add this only as an opt-in feature through a new binder level property - includeStoppedProcessorsForHealthCheck which is false by default to preserve the current health indicator behavior. Resolves https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 Resolves #1175 --- docs/src/main/asciidoc/kafka-streams.adoc | 6 ++++ .../kafka/streams/GlobalKTableBinder.java | 11 ++++++ .../binder/kafka/streams/KStreamBinder.java | 21 ++++++++++++ .../binder/kafka/streams/KTableBinder.java | 11 ++++++ .../KafkaStreamsBinderHealthIndicator.java | 30 ++++++++++++---- ...fkaStreamsBindingInformationCatalogue.java | 34 +++++++++++++++++++ ...aStreamsBinderConfigurationProperties.java | 9 +++++ 7 files changed, 116 insertions(+), 6 deletions(-) diff --git a/docs/src/main/asciidoc/kafka-streams.adoc b/docs/src/main/asciidoc/kafka-streams.adoc index ba59cf39d..9f016be24 100644 --- a/docs/src/main/asciidoc/kafka-streams.adoc +++ b/docs/src/main/asciidoc/kafka-streams.adoc @@ -2121,6 +2121,12 @@ Arbitrary consumer properties at the binder level. producerProperties:: Arbitrary producer properties at the binder level. +includeStoppedProcessorsForHealthCheck:: +When bindings for processors are stopped through actuator, then this processor will not participate in the health check by default. +Set this property to `true` to enable health check for all processors including the ones that are currently stopped through bindings actuator endpoint. ++ +Default: false + ==== Kafka Streams Producer Properties The following properties are _only_ available for Kafka Streams producers and must be prefixed with `spring.cloud.stream.kafka.streams.bindings..producer.` diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBinder.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBinder.java index 418bda59a..7269875da 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBinder.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBinder.java @@ -17,6 +17,7 @@ package org.springframework.cloud.stream.binder.kafka.streams; import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.kstream.GlobalKTable; import org.springframework.cloud.stream.binder.AbstractBinder; @@ -105,6 +106,12 @@ public class GlobalKTableBinder extends if (!streamsBuilderFactoryBean.isRunning()) { super.start(); GlobalKTableBinder.this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean); + //If we cached the previous KafkaStreams object (from a binding stop on the actuator), remove it. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG); + if (kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().containsKey(applicationId)) { + kafkaStreamsBindingInformationCatalogue.removePreviousKafkaStreamsForApplicationId(applicationId); + } } } @@ -115,6 +122,10 @@ public class GlobalKTableBinder extends super.stop(); GlobalKTableBinder.this.kafkaStreamsRegistry.unregisterKafkaStreams(kafkaStreams); KafkaStreamsBinderUtils.closeDlqProducerFactories(kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean); + //Caching the stopped KafkaStreams for health indicator purposes on the underlying processor. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + GlobalKTableBinder.this.kafkaStreamsBindingInformationCatalogue.addPreviousKafkaStreamsForApplicationId( + (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG), kafkaStreams); } } }; diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBinder.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBinder.java index e90cdbd1e..0e81494f1 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBinder.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBinder.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.processor.StreamPartitioner; @@ -134,6 +135,12 @@ class KStreamBinder extends if (!streamsBuilderFactoryBean.isRunning()) { super.start(); KStreamBinder.this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean); + //If we cached the previous KafkaStreams object (from a binding stop on the actuator), remove it. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG); + if (kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().containsKey(applicationId)) { + kafkaStreamsBindingInformationCatalogue.removePreviousKafkaStreamsForApplicationId(applicationId); + } } } @@ -144,6 +151,10 @@ class KStreamBinder extends super.stop(); KStreamBinder.this.kafkaStreamsRegistry.unregisterKafkaStreams(kafkaStreams); KafkaStreamsBinderUtils.closeDlqProducerFactories(kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean); + //Caching the stopped KafkaStreams for health indicator purposes on the underlying processor. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + KStreamBinder.this.kafkaStreamsBindingInformationCatalogue.addPreviousKafkaStreamsForApplicationId( + (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG), kafkaStreams); } } }; @@ -199,6 +210,12 @@ class KStreamBinder extends if (!streamsBuilderFactoryBean.isRunning()) { super.start(); KStreamBinder.this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean); + //If we cached the previous KafkaStreams object (from a binding stop on the actuator), remove it. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG); + if (kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().containsKey(applicationId)) { + kafkaStreamsBindingInformationCatalogue.removePreviousKafkaStreamsForApplicationId(applicationId); + } } } @@ -209,6 +226,10 @@ class KStreamBinder extends super.stop(); KStreamBinder.this.kafkaStreamsRegistry.unregisterKafkaStreams(kafkaStreams); KafkaStreamsBinderUtils.closeDlqProducerFactories(kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean); + //Caching the stopped KafkaStreams for health indicator purposes on the underlying processor + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + KStreamBinder.this.kafkaStreamsBindingInformationCatalogue.addPreviousKafkaStreamsForApplicationId( + (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG), kafkaStreams); } } }; diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBinder.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBinder.java index c82a520b4..a5dcd9c09 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBinder.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBinder.java @@ -17,6 +17,7 @@ package org.springframework.cloud.stream.binder.kafka.streams; import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.kstream.KTable; import org.springframework.cloud.stream.binder.AbstractBinder; @@ -106,6 +107,12 @@ class KTableBinder extends if (!streamsBuilderFactoryBean.isRunning()) { super.start(); KTableBinder.this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean); + //If we cached the previous KafkaStreams object (from a binding stop on the actuator), remove it. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG); + if (kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().containsKey(applicationId)) { + kafkaStreamsBindingInformationCatalogue.removePreviousKafkaStreamsForApplicationId(applicationId); + } } } @@ -116,6 +123,10 @@ class KTableBinder extends super.stop(); KTableBinder.this.kafkaStreamsRegistry.unregisterKafkaStreams(kafkaStreams); KafkaStreamsBinderUtils.closeDlqProducerFactories(kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean); + //Caching the stopped KafkaStreams for health indicator purposes on the underlying processor. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + KTableBinder.this.kafkaStreamsBindingInformationCatalogue.addPreviousKafkaStreamsForApplicationId( + (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG), kafkaStreams); } } }; diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderHealthIndicator.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderHealthIndicator.java index 08bc63dd6..6db6a64ce 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderHealthIndicator.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderHealthIndicator.java @@ -19,6 +19,7 @@ package org.springframework.cloud.stream.binder.kafka.streams; import java.lang.reflect.Method; import java.time.Duration; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -31,8 +32,8 @@ import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.ListTopicsResult; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.StreamsConfig; -import org.apache.kafka.streams.processor.TaskMetadata; -import org.apache.kafka.streams.processor.ThreadMetadata; +import org.apache.kafka.streams.TaskMetadata; +import org.apache.kafka.streams.ThreadMetadata; import org.springframework.beans.factory.DisposableBean; import org.springframework.boot.actuate.health.AbstractHealthIndicator; @@ -118,7 +119,12 @@ public class KafkaStreamsBinderHealthIndicator extends AbstractHealthIndicator i } else { boolean up = true; - for (KafkaStreams kStream : kafkaStreamsRegistry.getKafkaStreams()) { + final Set kafkaStreams = kafkaStreamsRegistry.getKafkaStreams(); + Set allKafkaStreams = new HashSet<>(kafkaStreams); + if (this.configurationProperties.isIncludeStoppedProcessorsForHealthCheck()) { + allKafkaStreams.addAll(kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().values()); + } + for (KafkaStreams kStream : allKafkaStreams) { if (isKafkaStreams25) { up &= kStream.state().isRunningOrRebalancing(); } @@ -156,7 +162,8 @@ public class KafkaStreamsBinderHealthIndicator extends AbstractHealthIndicator i } if (isRunningResult) { - for (ThreadMetadata metadata : kafkaStreams.localThreadsMetadata()) { + final Set threadMetadata = kafkaStreams.metadataForLocalThreads(); + for (ThreadMetadata metadata : threadMetadata) { perAppdIdDetails.put("threadName", metadata.threadName()); perAppdIdDetails.put("threadState", metadata.threadState()); perAppdIdDetails.put("adminClientId", metadata.adminClientId()); @@ -172,8 +179,19 @@ public class KafkaStreamsBinderHealthIndicator extends AbstractHealthIndicator i } else { final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsRegistry.streamBuilderFactoryBean(kafkaStreams); - final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG); - details.put(applicationId, String.format("The processor with application.id %s is down", applicationId)); + String applicationId = null; + if (streamsBuilderFactoryBean != null) { + applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG); + } + else { + final Map stoppedKafkaStreamsPerBinding = kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams(); + for (String appId : stoppedKafkaStreamsPerBinding.keySet()) { + if (stoppedKafkaStreamsPerBinding.get(appId).equals(kafkaStreams)) { + applicationId = appId; + } + } + } + details.put(applicationId, String.format("The processor with application.id %s is down. Current state: %s", applicationId, kafkaStreams.state())); } return details; } diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java index c12b77d07..92b856d6b 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java @@ -26,6 +26,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.kstream.KStream; @@ -62,6 +63,8 @@ public class KafkaStreamsBindingInformationCatalogue { private final Map bindingNamesPerTarget = new HashMap<>(); + private final Map previousKafkaStreamsPerApplicationId = new HashMap<>(); + private final Map>> dlqProducerFactories = new HashMap<>(); /** @@ -213,4 +216,35 @@ public class KafkaStreamsBindingInformationCatalogue { } producerFactories.add(producerFactory); } + + /** + * Caching the previous KafkaStreams for the applicaiton.id when binding is stopped through actuator. + * See https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + * + * @param applicationId application.id + * @param kafkaStreams {@link KafkaStreams} object + */ + public void addPreviousKafkaStreamsForApplicationId(String applicationId, KafkaStreams kafkaStreams) { + this.previousKafkaStreamsPerApplicationId.put(applicationId, kafkaStreams); + } + + /** + * Remove the previously cached KafkaStreams object. + * See https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + * + * @param applicationId application.id + */ + public void removePreviousKafkaStreamsForApplicationId(String applicationId) { + this.previousKafkaStreamsPerApplicationId.remove(applicationId); + } + + /** + * Get all stopped KafkaStreams objects through actuator binding stop. + * See https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + * + * @return stopped KafkaStreams objects map + */ + public Map getStoppedKafkaStreams() { + return this.previousKafkaStreamsPerApplicationId; + } } diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsBinderConfigurationProperties.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsBinderConfigurationProperties.java index 9aeb4c7d0..95b1c93bc 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsBinderConfigurationProperties.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsBinderConfigurationProperties.java @@ -74,6 +74,7 @@ public class KafkaStreamsBinderConfigurationProperties */ private DeserializationExceptionHandler deserializationExceptionHandler; + private boolean includeStoppedProcessorsForHealthCheck; public Map getFunctions() { return functions; @@ -127,6 +128,14 @@ public class KafkaStreamsBinderConfigurationProperties this.deserializationExceptionHandler = deserializationExceptionHandler; } + public boolean isIncludeStoppedProcessorsForHealthCheck() { + return includeStoppedProcessorsForHealthCheck; + } + + public void setIncludeStoppedProcessorsForHealthCheck(boolean includeStoppedProcessorsForHealthCheck) { + this.includeStoppedProcessorsForHealthCheck = includeStoppedProcessorsForHealthCheck; + } + public static class StateStoreRetry { private int maxAttempts = 1; From 921b47d1e4857e3385c0f4ea4b0aff714e5816c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduard=20Dom=C3=ADnguez?= Date: Fri, 10 Dec 2021 12:26:17 +0100 Subject: [PATCH 10/19] GH-1176: KeyValueSerdeResolver improvements Use extended properties when initializing Consumer and Producer Serdes. Updated copyright years and authors. Resolves https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1176 --- .../kafka/streams/KeyValueSerdeResolver.java | 52 ++++++++++++------- .../KafkaStreamsBinderBootstrapTest.java | 19 ++++++- 2 files changed, 49 insertions(+), 22 deletions(-) diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KeyValueSerdeResolver.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KeyValueSerdeResolver.java index 3d24433de..fc925dc9d 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KeyValueSerdeResolver.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KeyValueSerdeResolver.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 the original author or authors. + * Copyright 2018-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ package org.springframework.cloud.stream.binder.kafka.streams; import java.lang.reflect.Method; import java.util.Arrays; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -69,6 +70,7 @@ import org.springframework.util.StringUtils; * * @author Soby Chacko * @author Lei Chen + * @author Eduard Domínguez */ public class KeyValueSerdeResolver implements ApplicationContextAware { @@ -96,14 +98,14 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { KafkaStreamsConsumerProperties extendedConsumerProperties) { String keySerdeString = extendedConsumerProperties.getKeySerde(); - return getKeySerde(keySerdeString); + return getKeySerde(keySerdeString, extendedConsumerProperties.getConfiguration()); } public Serde getInboundKeySerde( KafkaStreamsConsumerProperties extendedConsumerProperties, ResolvableType resolvableType) { String keySerdeString = extendedConsumerProperties.getKeySerde(); - return getKeySerde(keySerdeString, resolvableType); + return getKeySerde(keySerdeString, resolvableType, extendedConsumerProperties.getConfiguration()); } /** @@ -120,7 +122,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { String valueSerdeString = extendedConsumerProperties.getValueSerde(); try { if (consumerProperties != null && consumerProperties.isUseNativeDecoding()) { - valueSerde = getValueSerde(valueSerdeString); + valueSerde = getValueSerde(valueSerdeString, extendedConsumerProperties.getConfiguration()); } else { valueSerde = Serdes.ByteArray(); @@ -140,7 +142,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { String valueSerdeString = extendedConsumerProperties.getValueSerde(); try { if (consumerProperties != null && consumerProperties.isUseNativeDecoding()) { - valueSerde = getValueSerde(valueSerdeString, resolvableType); + valueSerde = getValueSerde(valueSerdeString, resolvableType, extendedConsumerProperties.getConfiguration()); } else { valueSerde = Serdes.ByteArray(); @@ -158,11 +160,11 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { * @return configurd {@link Serde} for the outbound key. */ public Serde getOuboundKeySerde(KafkaStreamsProducerProperties properties) { - return getKeySerde(properties.getKeySerde()); + return getKeySerde(properties.getKeySerde(), properties.getConfiguration()); } public Serde getOuboundKeySerde(KafkaStreamsProducerProperties properties, ResolvableType resolvableType) { - return getKeySerde(properties.getKeySerde(), resolvableType); + return getKeySerde(properties.getKeySerde(), resolvableType, properties.getConfiguration()); } @@ -179,7 +181,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { try { if (producerProperties.isUseNativeEncoding()) { valueSerde = getValueSerde( - kafkaStreamsProducerProperties.getValueSerde()); + kafkaStreamsProducerProperties.getValueSerde(), kafkaStreamsProducerProperties.getConfiguration()); } else { valueSerde = Serdes.ByteArray(); @@ -197,7 +199,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { try { if (producerProperties.isUseNativeEncoding()) { valueSerde = getValueSerde( - kafkaStreamsProducerProperties.getValueSerde(), resolvableType); + kafkaStreamsProducerProperties.getValueSerde(), resolvableType, kafkaStreamsProducerProperties.getConfiguration()); } else { valueSerde = Serdes.ByteArray(); @@ -215,7 +217,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { * @return {@link Serde} for the state store key. */ public Serde getStateStoreKeySerde(String keySerdeString) { - return getKeySerde(keySerdeString); + return getKeySerde(keySerdeString, (Map) null); } /** @@ -225,14 +227,14 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { */ public Serde getStateStoreValueSerde(String valueSerdeString) { try { - return getValueSerde(valueSerdeString); + return getValueSerde(valueSerdeString, (Map) null); } catch (ClassNotFoundException ex) { throw new IllegalStateException("Serde class not found: ", ex); } } - private Serde getKeySerde(String keySerdeString) { + private Serde getKeySerde(String keySerdeString, Map extendedConfiguration) { Serde keySerde; try { if (StringUtils.hasText(keySerdeString)) { @@ -241,8 +243,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { else { keySerde = getFallbackSerde("default.key.serde"); } - keySerde.configure(this.streamConfigGlobalProperties, true); - + keySerde.configure(combineStreamConfigProperties(extendedConfiguration), false); } catch (ClassNotFoundException ex) { throw new IllegalStateException("Serde class not found: ", ex); @@ -250,7 +251,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { return keySerde; } - private Serde getKeySerde(String keySerdeString, ResolvableType resolvableType) { + private Serde getKeySerde(String keySerdeString, ResolvableType resolvableType, Map extendedConfiguration) { Serde keySerde = null; try { if (StringUtils.hasText(keySerdeString)) { @@ -267,7 +268,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { keySerde = Serdes.ByteArray(); } } - keySerde.configure(this.streamConfigGlobalProperties, true); + keySerde.configure(combineStreamConfigProperties(extendedConfiguration), false); } catch (ClassNotFoundException ex) { throw new IllegalStateException("Serde class not found: ", ex); @@ -380,7 +381,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { } - private Serde getValueSerde(String valueSerdeString) + private Serde getValueSerde(String valueSerdeString, Map extendedConfiguration) throws ClassNotFoundException { Serde valueSerde; if (StringUtils.hasText(valueSerdeString)) { @@ -389,7 +390,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { else { valueSerde = getFallbackSerde("default.value.serde"); } - valueSerde.configure(this.streamConfigGlobalProperties, false); + valueSerde.configure(combineStreamConfigProperties(extendedConfiguration), false); return valueSerde; } @@ -403,7 +404,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { } @SuppressWarnings("unchecked") - private Serde getValueSerde(String valueSerdeString, ResolvableType resolvableType) + private Serde getValueSerde(String valueSerdeString, ResolvableType resolvableType, Map extendedConfiguration) throws ClassNotFoundException { Serde valueSerde = null; if (StringUtils.hasText(valueSerdeString)) { @@ -422,7 +423,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { valueSerde = Serdes.ByteArray(); } } - valueSerde.configure(streamConfigGlobalProperties, false); + valueSerde.configure(combineStreamConfigProperties(extendedConfiguration), false); return valueSerde; } @@ -430,4 +431,15 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { context = (ConfigurableApplicationContext) applicationContext; } + + private Map combineStreamConfigProperties(Map extendedConfiguration) { + if (extendedConfiguration != null && !extendedConfiguration.isEmpty()) { + Map streamConfiguration = new HashMap(this.streamConfigGlobalProperties); + streamConfiguration.putAll(extendedConfiguration); + return streamConfiguration; + } + else { + return this.streamConfigGlobalProperties; + } + } } diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderBootstrapTest.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderBootstrapTest.java index ee572f870..3bf29b698 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderBootstrapTest.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderBootstrapTest.java @@ -20,6 +20,9 @@ import java.util.Map; import java.util.Properties; import java.util.function.Consumer; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.type.TypeFactory; +import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.security.JaasUtils; import org.apache.kafka.streams.kstream.GlobalKTable; import org.apache.kafka.streams.kstream.KStream; @@ -28,9 +31,11 @@ import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; +import org.springframework.beans.DirectFieldAccessor; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.stream.binder.kafka.streams.KeyValueSerdeResolver; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.kafka.config.StreamsBuilderFactoryBean; @@ -40,6 +45,7 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat; /** * @author Soby Chacko + * @author Eduard Domínguez */ public class KafkaStreamsBinderBootstrapTest { @@ -111,7 +117,7 @@ public class KafkaStreamsBinderBootstrapTest { + "=testKafkaStreamsBinderWithStandardConfigurationCanStart", "--spring.cloud.stream.kafka.streams.bindings.input2-in-0.consumer.application-id" + "=testKafkaStreamsBinderWithStandardConfigurationCanStart-foo", - "--spring.cloud.stream.kafka.streams.bindings.input2-in-0.consumer.configuration.spring.json.value.type.method=com.test.MyClass", + "--spring.cloud.stream.kafka.streams.bindings.input2-in-0.consumer.configuration.spring.json.value.type.method=" + this.getClass().getName() + ".determineType", "--spring.cloud.stream.kafka.streams.bindings.input3-in-0.consumer.application-id" + "=testKafkaStreamsBinderWithStandardConfigurationCanStart-foobar", "--spring.cloud.stream.kafka.streams.binder.brokers=" @@ -134,10 +140,19 @@ public class KafkaStreamsBinderBootstrapTest { final StreamsBuilderFactoryBean input3SBFB = applicationContext.getBean("&stream-builder-input3", StreamsBuilderFactoryBean.class); final Properties streamsConfiguration3 = input3SBFB.getStreamsConfiguration(); assertThat(streamsConfiguration3.containsKey("spring.json.value.type.method")).isFalse(); + applicationContext.getBean(KeyValueSerdeResolver.class); + String configuredSerdeTypeResolver = (String) new DirectFieldAccessor(input2SBFB.getKafkaStreams()) + .getPropertyValue("taskTopology.processorNodes[0].valDeserializer.typeResolver.arg$2"); + + assertThat(this.getClass().getName() + ".determineType").isEqualTo(configuredSerdeTypeResolver); applicationContext.close(); } + public static JavaType determineType(byte[] data, Headers headers) { + return TypeFactory.defaultInstance().constructParametricType(Map.class, String.class, String.class); + } + @SpringBootApplication static class SimpleKafkaStreamsApplication { @@ -149,7 +164,7 @@ public class KafkaStreamsBinderBootstrapTest { } @Bean - public Consumer> input2() { + public Consumer>> input2() { return s -> { // No-op consumer }; From fb03d2ae8ec543736c6febbcc10c6428e045e4f7 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Mon, 3 Jan 2022 18:32:13 -0500 Subject: [PATCH 11/19] Version upgrades 4.0.0-SNAPSHOT Spring Kafka - 3.0.0-SNAPSHOT Spring Integraton Kafka - 6.0.0-SNAPSHOT Spring Cloud Stream - 4.0.0-SNAPSHOT Code changes for Jakarta --- docs/pom.xml | 2 +- pom.xml | 12 ++++++------ spring-cloud-starter-stream-kafka/pom.xml | 2 +- spring-cloud-stream-binder-kafka-core/pom.xml | 2 +- .../KafkaBinderConfigurationProperties.java | 7 +++---- .../kafka/properties/KafkaProducerProperties.java | 2 +- spring-cloud-stream-binder-kafka-streams/pom.xml | 2 +- spring-cloud-stream-binder-kafka/pom.xml | 2 +- 8 files changed, 15 insertions(+), 16 deletions(-) diff --git a/docs/pom.xml b/docs/pom.xml index 1778c636d..4c5528330 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -7,7 +7,7 @@ org.springframework.cloud spring-cloud-stream-binder-kafka-parent - 3.2.0-SNAPSHOT + 4.0.0-SNAPSHOT jar spring-cloud-stream-binder-kafka-docs diff --git a/pom.xml b/pom.xml index b60bbae3d..732fa121e 100644 --- a/pom.xml +++ b/pom.xml @@ -2,12 +2,12 @@ 4.0.0 spring-cloud-stream-binder-kafka-parent - 3.2.0-SNAPSHOT + 4.0.0-SNAPSHOT pom org.springframework.cloud spring-cloud-build - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT @@ -20,11 +20,11 @@ HEAD - 1.8 - 2.8.0-RC1 - 5.5.5 + 17 + 3.0.0-SNAPSHOT + 6.0.0-SNAPSHOT 3.0.0 - 3.2.0-SNAPSHOT + 4.0.0-SNAPSHOT true true true diff --git a/spring-cloud-starter-stream-kafka/pom.xml b/spring-cloud-starter-stream-kafka/pom.xml index 42f3042b9..0959fbc20 100644 --- a/spring-cloud-starter-stream-kafka/pom.xml +++ b/spring-cloud-starter-stream-kafka/pom.xml @@ -4,7 +4,7 @@ org.springframework.cloud spring-cloud-stream-binder-kafka-parent - 3.2.0-SNAPSHOT + 4.0.0-SNAPSHOT spring-cloud-starter-stream-kafka Spring Cloud Starter Stream Kafka diff --git a/spring-cloud-stream-binder-kafka-core/pom.xml b/spring-cloud-stream-binder-kafka-core/pom.xml index d550a1495..51330e69b 100644 --- a/spring-cloud-stream-binder-kafka-core/pom.xml +++ b/spring-cloud-stream-binder-kafka-core/pom.xml @@ -5,7 +5,7 @@ org.springframework.cloud spring-cloud-stream-binder-kafka-parent - 3.2.0-SNAPSHOT + 4.0.0-SNAPSHOT spring-cloud-stream-binder-kafka-core Spring Cloud Stream Kafka Binder Core diff --git a/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java b/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java index c53566eef..b8ee4d351 100644 --- a/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java +++ b/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java @@ -28,10 +28,9 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import javax.validation.constraints.AssertTrue; -import javax.validation.constraints.Min; -import javax.validation.constraints.NotNull; - +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.kafka.clients.consumer.ConsumerConfig; diff --git a/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaProducerProperties.java b/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaProducerProperties.java index 3653c5f61..54ca36713 100644 --- a/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaProducerProperties.java +++ b/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaProducerProperties.java @@ -19,7 +19,7 @@ package org.springframework.cloud.stream.binder.kafka.properties; import java.util.HashMap; import java.util.Map; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; import org.springframework.expression.Expression; diff --git a/spring-cloud-stream-binder-kafka-streams/pom.xml b/spring-cloud-stream-binder-kafka-streams/pom.xml index f68b11e69..24e93421d 100644 --- a/spring-cloud-stream-binder-kafka-streams/pom.xml +++ b/spring-cloud-stream-binder-kafka-streams/pom.xml @@ -10,7 +10,7 @@ org.springframework.cloud spring-cloud-stream-binder-kafka-parent - 3.2.0-SNAPSHOT + 4.0.0-SNAPSHOT diff --git a/spring-cloud-stream-binder-kafka/pom.xml b/spring-cloud-stream-binder-kafka/pom.xml index df027c2af..de5b7b519 100644 --- a/spring-cloud-stream-binder-kafka/pom.xml +++ b/spring-cloud-stream-binder-kafka/pom.xml @@ -10,7 +10,7 @@ org.springframework.cloud spring-cloud-stream-binder-kafka-parent - 3.2.0-SNAPSHOT + 4.0.0-SNAPSHOT From 1cdfb962c9f7906b0b724e08df94902ffa415bfd Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Mon, 3 Jan 2022 18:55:22 -0500 Subject: [PATCH 12/19] checkstyle fix --- .../streams/KafkaStreamsInteractiveQueryIntegrationTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java index aff83c3a5..83ab43ef0 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java @@ -18,8 +18,8 @@ package org.springframework.cloud.stream.binder.kafka.streams; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.Properties; +import java.util.function.Function; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; From d1a9eab14bf83393c513add3b26b10f25c4ffe9a Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Tue, 4 Jan 2022 14:53:40 -0500 Subject: [PATCH 13/19] Default maven-antrun version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 732fa121e..cb45e73dc 100644 --- a/pom.xml +++ b/pom.xml @@ -144,7 +144,7 @@ org.apache.maven.plugins maven-antrun-plugin - 1.7 + org.apache.maven.plugins From 3cc3680f63d952555dc9b87ab0d620041e8ebcb8 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Wed, 5 Jan 2022 19:30:34 -0500 Subject: [PATCH 14/19] Event type routing improvements (Kafka Streams) When routing by event types, the deserializer omits the topic and header information. Fixing this issue. Resolves https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1186 --- .../AbstractKafkaStreamsBinderProcessor.java | 44 +++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/AbstractKafkaStreamsBinderProcessor.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/AbstractKafkaStreamsBinderProcessor.java index 21480f0f1..9c787b58b 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/AbstractKafkaStreamsBinderProcessor.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/AbstractKafkaStreamsBinderProcessor.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2021 the original author or authors. + * Copyright 2019-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,9 @@ package org.springframework.cloud.stream.binder.kafka.streams; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; import org.apache.commons.logging.Log; @@ -40,9 +42,10 @@ import org.apache.kafka.streams.kstream.GlobalKTable; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.Materialized; -import org.apache.kafka.streams.processor.Processor; -import org.apache.kafka.streams.processor.ProcessorContext; import org.apache.kafka.streams.processor.TimestampExtractor; +import org.apache.kafka.streams.processor.api.Processor; +import org.apache.kafka.streams.processor.api.Record; +import org.apache.kafka.streams.processor.api.RecordMetadata; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.StoreBuilder; @@ -449,12 +452,15 @@ public abstract class AbstractKafkaStreamsBinderProcessor implements Application //See this issue for more context: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1003 if (StringUtils.hasText(kafkaStreamsConsumerProperties.getEventTypes())) { AtomicBoolean matched = new AtomicBoolean(); + AtomicReference topicObject = new AtomicReference<>(); + AtomicReference headersObject = new AtomicReference<>(); // Processor to retrieve the header value. - stream.process(() -> eventTypeProcessor(kafkaStreamsConsumerProperties, matched)); + stream.process(() -> eventTypeProcessor(kafkaStreamsConsumerProperties, matched, topicObject, headersObject)); // Branching based on event type match. final KStream[] branch = stream.branch((key, value) -> matched.getAndSet(false)); // Deserialize if we have a branch from above. - final KStream deserializedKStream = branch[0].mapValues(value -> valueSerde.deserializer().deserialize(null, ((Bytes) value).get())); + final KStream deserializedKStream = branch[0].mapValues(value -> valueSerde.deserializer().deserialize( + topicObject.get(), headersObject.get(), ((Bytes) value).get())); return getkStream(bindingProperties, deserializedKStream, nativeDecoding); } return getkStream(bindingProperties, stream, nativeDecoding); @@ -549,14 +555,18 @@ public abstract class AbstractKafkaStreamsBinderProcessor implements Application consumed); if (StringUtils.hasText(kafkaStreamsConsumerProperties.getEventTypes())) { AtomicBoolean matched = new AtomicBoolean(); + AtomicReference topicObject = new AtomicReference<>(); + AtomicReference headersObject = new AtomicReference<>(); + final KStream stream = kTable.toStream(); // Processor to retrieve the header value. - stream.process(() -> eventTypeProcessor(kafkaStreamsConsumerProperties, matched)); + stream.process(() -> eventTypeProcessor(kafkaStreamsConsumerProperties, matched, topicObject, headersObject)); // Branching based on event type match. final KStream[] branch = stream.branch((key, value) -> matched.getAndSet(false)); // Deserialize if we have a branch from above. - final KStream deserializedKStream = branch[0].mapValues(value -> valueSerde.deserializer().deserialize(null, ((Bytes) value).get())); + final KStream deserializedKStream = branch[0].mapValues(value -> valueSerde.deserializer().deserialize( + topicObject.get(), headersObject.get(), ((Bytes) value).get())); return deserializedKStream.toTable(); } @@ -581,19 +591,27 @@ public abstract class AbstractKafkaStreamsBinderProcessor implements Application return consumed; } - private Processor eventTypeProcessor(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, AtomicBoolean matched) { - return new Processor() { + private Processor eventTypeProcessor(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, + AtomicBoolean matched, AtomicReference topicObject, AtomicReference headersObject) { + return new Processor() { - ProcessorContext context; + org.apache.kafka.streams.processor.api.ProcessorContext context; @Override - public void init(ProcessorContext context) { + public void init(org.apache.kafka.streams.processor.api.ProcessorContext context) { + Processor.super.init(context); this.context = context; } @Override - public void process(Object key, Object value) { - final Headers headers = this.context.headers(); + public void process(Record record) { + final Headers headers = record.headers(); + headersObject.set(headers); + final Optional optional = this.context.recordMetadata(); + if (optional.isPresent()) { + final RecordMetadata recordMetadata = optional.get(); + topicObject.set(recordMetadata.topic()); + } final Iterable
eventTypeHeader = headers.headers(kafkaStreamsConsumerProperties.getEventTypeHeaderKey()); if (eventTypeHeader != null && eventTypeHeader.iterator().hasNext()) { String eventTypeFromHeader = new String(eventTypeHeader.iterator().next().value()); From da9bc354e443a71a9bfc077b8ebd1876c59839d2 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Fri, 7 Jan 2022 20:04:10 -0500 Subject: [PATCH 15/19] StreamListener docs cleanup. Fixing Kafka Streams composition tests due to an application.id issue. --- docs/src/main/asciidoc/kafka-streams.adoc | 252 +----------------- docs/src/main/asciidoc/overview.adoc | 2 - .../KafkaStreamsFunctionCompositionTests.java | 2 + .../config/KafkaBinderConfiguration.java | 2 - 4 files changed, 10 insertions(+), 248 deletions(-) diff --git a/docs/src/main/asciidoc/kafka-streams.adoc b/docs/src/main/asciidoc/kafka-streams.adoc index 9f016be24..be68b9621 100644 --- a/docs/src/main/asciidoc/kafka-streams.adoc +++ b/docs/src/main/asciidoc/kafka-streams.adoc @@ -430,213 +430,6 @@ public Function, KStream> bar() { You can compose them as `foo|bar`, but keep in mind that the second function (`bar` in this case) must have a `KTable` as input since the first function (`foo`) has `KTable` as output. -==== Imperative programming model. - -Starting with `3.1.0` version of the binder, we recommend using the functional programming model described above for Kafka Streams binder based applications. -The support for `StreamListener` is deprecated starting with `3.1.0` of Spring Cloud Stream. -Below, we are providing some details on the `StreamListener` based Kafka Streams processors as a reference. - -Following is the equivalent of the Word count example using `StreamListener`. - -[source] ----- -@SpringBootApplication -@EnableBinding(KafkaStreamsProcessor.class) -public class WordCountProcessorApplication { - - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - return input - .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) - .groupBy((key, value) -> value) - .windowedBy(TimeWindows.of(5000)) - .count(Materialized.as("WordCounts-multi")) - .toStream() - .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))); - } - - public static void main(String[] args) { - SpringApplication.run(WordCountProcessorApplication.class, args); - } ----- - -As you can see, this is a bit more verbose since you need to provide `EnableBinding` and the other extra annotations like `StreamListener` and `SendTo` to make it a complete application. -`EnableBinding` is where you specify your binding interface that contains your bindings. -In this case, we are using the stock `KafkaStreamsProcessor` binding interface that has the following contracts. - -[source] ----- -public interface KafkaStreamsProcessor { - - @Input("input") - KStream input(); - - @Output("output") - KStream output(); - -} ----- - -Binder will create bindings for the input `KStream` and output `KStream` since you are using a binding interface that contains those declarations. - -In addition to the obvious differences in the programming model offered in the functional style, one particular thing that needs to be mentioned here is that the binding names are what you specify in the binding interface. -For example, in the above application, since we are using `KafkaStreamsProcessor`, the binding names are `input` and `output`. -Binding properties need to use those names. For instance `spring.cloud.stream.bindings.input.destination`, `spring.cloud.stream.bindings.output.destination` etc. -Keep in mind that this is fundamentally different from the functional style since there the binder generates binding names for the application. -This is because the application does not provide any binding interfaces in the functional model using `EnableBinding`. - -Here is another example of a sink where we have two inputs. - -[source] ----- -@EnableBinding(KStreamKTableBinding.class) -..... -..... -@StreamListener -public void process(@Input("inputStream") KStream playEvents, - @Input("inputTable") KTable songTable) { - .... - .... -} - -interface KStreamKTableBinding { - - @Input("inputStream") - KStream inputStream(); - - @Input("inputTable") - KTable inputTable(); -} - ----- - -Following is the `StreamListener` equivalent of the same `BiFunction` based processor that we saw above. - - -[source] ----- -@EnableBinding(KStreamKTableBinding.class) -.... -.... - -@StreamListener -@SendTo("output") -public KStream process(@Input("input") KStream userClicksStream, - @Input("inputTable") KTable userRegionsTable) { -.... -.... -} - -interface KStreamKTableBinding extends KafkaStreamsProcessor { - - @Input("inputX") - KTable inputTable(); -} ----- - -Finally, here is the `StreamListener` equivalent of the application with three inputs and curried functions. - -[source] ----- -@EnableBinding(CustomGlobalKTableProcessor.class) -... -... - @StreamListener - @SendTo("output") - public KStream process( - @Input("input-1") KStream ordersStream, - @Input("input-2") GlobalKTable customers, - @Input("input-3") GlobalKTable products) { - - KStream customerOrdersStream = ordersStream.join( - customers, (orderId, order) -> order.getCustomerId(), - (order, customer) -> new CustomerOrder(customer, order)); - - return customerOrdersStream.join(products, - (orderId, customerOrder) -> customerOrder.productId(), - (customerOrder, product) -> { - EnrichedOrder enrichedOrder = new EnrichedOrder(); - enrichedOrder.setProduct(product); - enrichedOrder.setCustomer(customerOrder.customer); - enrichedOrder.setOrder(customerOrder.order); - return enrichedOrder; - }); - } - - interface CustomGlobalKTableProcessor { - - @Input("input-1") - KStream input1(); - - @Input("input-2") - GlobalKTable input2(); - - @Input("input-3") - GlobalKTable input3(); - - @Output("output") - KStream output(); - } - ----- - -You might notice that the above two examples are even more verbose since in addition to provide `EnableBinding`, you also need to write your own custom binding interface as well. -Using the functional model, you can avoid all those ceremonial details. - -Before we move on from looking at the general programming model offered by Kafka Streams binder, here is the `StreamListener` version of multiple output bindings. - -[source] ----- -EnableBinding(KStreamProcessorWithBranches.class) -public static class WordCountProcessorApplication { - - @Autowired - private TimeWindows timeWindows; - - @StreamListener("input") - @SendTo({"output1","output2","output3"}) - public KStream[] process(KStream input) { - - Predicate isEnglish = (k, v) -> v.word.equals("english"); - Predicate isFrench = (k, v) -> v.word.equals("french"); - Predicate isSpanish = (k, v) -> v.word.equals("spanish"); - - return input - .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) - .groupBy((key, value) -> value) - .windowedBy(timeWindows) - .count(Materialized.as("WordCounts-1")) - .toStream() - .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))) - .branch(isEnglish, isFrench, isSpanish); - } - - interface KStreamProcessorWithBranches { - - @Input("input") - KStream input(); - - @Output("output1") - KStream output1(); - - @Output("output2") - KStream output2(); - - @Output("output3") - KStream output3(); - } -} ----- - -To recap, we have reviewed the various programming model choices when using the Kafka Streams binder. - -The binder provides binding capabilities for `KStream`, `KTable` and `GlobalKTable` on the input. -`KTable` and `GlobalKTable` bindings are only available on the input. -Binder supports both input and output bindings for `KStream`. - -The upshot of the programming model of Kafka Streams binder is that the binder provides you the flexibility of going with a fully functional programming model or using the `StreamListener` based imperative approach. - === Ancillaries to the programming model ==== Multiple Kafka Streams processors within a single application @@ -677,7 +470,7 @@ This is also true when you have a single Kafka Streams processor and other types Application id is a mandatory property that you need to provide for a Kafka Streams application. Spring Cloud Stream Kafka Streams binder allows you to configure this application id in multiple ways. -If you only have one single processor or `StreamListener` in the application, then you can set this at the binder level using the following property: +If you only have one single processor in the application, then you can set this at the binder level using the following property: `spring.cloud.stream.kafka.streams.binder.applicationId`. @@ -712,33 +505,6 @@ and `spring.cloud.stream.kafka.streams.binder.functions.anotherProcess.applicationId` -In the case of `StreamListener`, you need to set this on the first input binding on the processor. - -For e.g. imagine that you have the following two `StreamListener` based processors. - -``` -@StreamListener -@SendTo("output") -public KStream process(@Input("input") > input) { - ... -} - -@StreamListener -@SendTo("anotherOutput") -public KStream anotherProcess(@Input("anotherInput") > input) { - ... -} -``` - -Then you must set the application id for this using the following binding property. - -`spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId` - -and - -`spring.cloud.stream.kafka.streams.bindings.anotherInput.consumer.applicationId` - - For function based model also, this approach of setting application id at the binding level will work. However, setting per function at the binder level as we have seen above is much easier if you are using the functional model. @@ -749,14 +515,12 @@ If the application does not provide an application ID, then in that case the bin This is convenient in development scenarios as it avoids the need for explicitly providing the application ID. The generated application ID in this manner will be static over application restarts. In the case of functional model, the generated application ID will be the function bean name followed by the literal `applicationID`, for e.g `process-applicationID` if `process` if the function bean name. -In the case of `StreamListener`, instead of using the function bean name, the generated application ID will be use the containing class name followed by the method name followed by the literal `applicationId`. ====== Summary of setting Application ID -* By default, binder will auto generate the application ID per function or `StreamListener` methods. +* By default, binder will auto generate the application ID per function methods. * If you have a single processor, then you can use `spring.kafka.streams.applicationId`, `spring.application.name` or `spring.cloud.stream.kafka.streams.binder.applicationId`. * If you have multiple processors, then application ID can be set per function using the property - `spring.cloud.stream.kafka.streams.binder.functions..applicationId`. -In the case of `StreamListener`, this can be done using `spring.cloud.stream.kafka.streams.bindings.input.applicationId`, assuming that the input binding name is `input`. ==== Overriding the default binding names generated by the binder with the functional style @@ -816,7 +580,7 @@ Keys are always deserialized using native Serdes. For values, by default, deserialization on the inbound is natively performed by Kafka. Please note that this is a major change on default behavior from previous versions of Kafka Streams binder where the deserialization was done by the framework. -Kafka Streams binder will try to infer matching `Serde` types by looking at the type signature of `java.util.function.Function|Consumer` or `StreamListener`. +Kafka Streams binder will try to infer matching `Serde` types by looking at the type signature of `java.util.function.Function|Consumer`. Here is the order that it matches the Serdes. * If the application provides a bean of type `Serde` and if the return type is parameterized with the actual type of the incoming key or value type, then it will use that `Serde` for inbound deserialization. @@ -1016,7 +780,7 @@ It is always recommended to explicitly create a DLQ topic for each input binding ==== DLQ per input consumer binding The property `spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler` is applicable for the entire application. -This implies that if there are multiple functions or `StreamListener` methods in the same application, this property is applied to all of them. +This implies that if there are multiple functions in the same application, this property is applied to all of them. However, if you have multiple processors or multiple input bindings within a single processor, then you can use the finer-grained DLQ control that the binder provides per input consumer binding. If you have the following processor, @@ -1061,7 +825,7 @@ If you set a consumer binding's `dlqPartitions` property to a value greater than A couple of things to keep in mind when using the exception handling feature in Kafka Streams binder. * The property `spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler` is applicable for the entire application. -This implies that if there are multiple functions or `StreamListener` methods in the same application, this property is applied to all of them. +This implies that if there are multiple functions in the same application, this property is applied to all of them. * The exception handling for deserialization works consistently with native deserialization and framework provided message conversion. ==== Handling Production Exceptions in the Binder @@ -2100,7 +1864,7 @@ Default: `logAndFail` applicationId:: Convenient way to set the application.id for the Kafka Streams application globally at the binder level. -If the application contains multiple functions or `StreamListener` methods, then the application id should be set differently. +If the application contains multiple functions, then the application id should be set differently. See above where setting the application id is discussed in detail. + Default: application will generate a static application ID. See the application ID section for more details. @@ -2164,7 +1928,7 @@ The following properties are available for Kafka Streams consumers and must be p For convenience, if there are multiple input bindings and they all require a common value, that can be configured by using the prefix `spring.cloud.stream.kafka.streams.default.consumer.`. applicationId:: -Setting application.id per input binding. This is only preferred for `StreamListener` based processors, for function based processors see other approaches outlined above. +Setting application.id per input binding. + Default: See above. @@ -2238,7 +2002,7 @@ In Kafka Streams, you can control of the number of threads a processor can creat This, you can do using the various `configuration` options described above under binder, functions, producer or consumer level. You can also use the `concurrency` property that core Spring Cloud Stream provides for this purpose. When using this, you need to use it on the consumer. -When you have more than one input bindings either in a function or `StreamListener`, set this on the first input binding. +When you have more than one input binding, set this on the first input binding. For e.g. when setting `spring.cloud.stream.bindings.process-in-0.consumer.concurrency`, it will be translated as `num.stream.threads` by the binder. If you have multiple processors and one processor defines binding level concurrency, but not the others, those ones with no binding level concurrency will default back to the binder wide property specified through `spring.cloud.stream.kafka.streams.binder.configuration.num.stream.threads`. diff --git a/docs/src/main/asciidoc/overview.adoc b/docs/src/main/asciidoc/overview.adoc index 096eccf64..b3dca2399 100644 --- a/docs/src/main/asciidoc/overview.adoc +++ b/docs/src/main/asciidoc/overview.adoc @@ -364,8 +364,6 @@ Starting with version 3.0, when `spring.cloud.stream.binding..consumer.bat Otherwise, the method will be called with one record at a time. The size of the batch is controlled by Kafka consumer properties `max.poll.records`, `fetch.min.bytes`, `fetch.max.wait.ms`; refer to the Kafka documentation for more information. -Bear in mind that batch mode is not supported with `@StreamListener` - it only works with the newer functional programming model. - IMPORTANT: Retry within the binder is not supported when using batch mode, so `maxAttempts` will be overridden to 1. You can configure a `SeekToCurrentBatchErrorHandler` (using a `ListenerContainerCustomizer`) to achieve similar functionality to retry in the binder. You can also use a manual `AckMode` and call `Ackowledgment.nack(index, sleep)` to commit the offsets for a partial batch and have the remaining records redelivered. diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionCompositionTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionCompositionTests.java index f3716d414..cebe3121a 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionCompositionTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionCompositionTests.java @@ -224,6 +224,7 @@ public class KafkaStreamsFunctionCompositionTests { try (ConfigurableApplicationContext context = app.run( "--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.kafka.streams.binder.applicationId=my-app-id", "--spring.cloud.stream.function.definition=fooBiFunc|anotherFooFunc|yetAnotherFooFunc|lastFunctionInChain", "--spring.cloud.stream.function.bindings.fooBiFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-in-0=input1", "--spring.cloud.stream.function.bindings.fooBiFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-in-1=input2", @@ -266,6 +267,7 @@ public class KafkaStreamsFunctionCompositionTests { try (ConfigurableApplicationContext context = app.run( "--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.kafka.streams.binder.applicationId=my-app-id-xyz", "--spring.cloud.stream.function.definition=curriedFunc|anotherFooFunc|yetAnotherFooFunc|lastFunctionInChain", "--spring.cloud.stream.function.bindings.curriedFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-in-0=input1", "--spring.cloud.stream.function.bindings.curriedFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-in-1=input2", diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java index b8d03225c..c32de1134 100644 --- a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java @@ -28,7 +28,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; import org.springframework.boot.autoconfigure.kafka.KafkaProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.stream.annotation.StreamMessageConverter; import org.springframework.cloud.stream.binder.Binder; import org.springframework.cloud.stream.binder.kafka.KafkaBinderMetrics; import org.springframework.cloud.stream.binder.kafka.KafkaBindingRebalanceListener; @@ -141,7 +140,6 @@ public class KafkaBinderConfiguration { } @Bean - @StreamMessageConverter @ConditionalOnMissingBean(KafkaNullConverter.class) MessageConverter kafkaNullConverter() { return new KafkaNullConverter(); From e512b7a2c600d791f54d5dd2aa7be547f86c7abb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduard=20Dom=C3=ADnguez?= Date: Fri, 10 Dec 2021 12:26:17 +0100 Subject: [PATCH 16/19] Fix: KeySerde setup not using expected key type headers checkstyle fixes --- .../binder/kafka/streams/KeyValueSerdeResolver.java | 4 ++-- .../bootstrap/KafkaStreamsBinderBootstrapTest.java | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KeyValueSerdeResolver.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KeyValueSerdeResolver.java index fc925dc9d..b23d51393 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KeyValueSerdeResolver.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KeyValueSerdeResolver.java @@ -243,7 +243,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { else { keySerde = getFallbackSerde("default.key.serde"); } - keySerde.configure(combineStreamConfigProperties(extendedConfiguration), false); + keySerde.configure(combineStreamConfigProperties(extendedConfiguration), true); } catch (ClassNotFoundException ex) { throw new IllegalStateException("Serde class not found: ", ex); @@ -268,7 +268,7 @@ public class KeyValueSerdeResolver implements ApplicationContextAware { keySerde = Serdes.ByteArray(); } } - keySerde.configure(combineStreamConfigProperties(extendedConfiguration), false); + keySerde.configure(combineStreamConfigProperties(extendedConfiguration), true); } catch (ClassNotFoundException ex) { throw new IllegalStateException("Serde class not found: ", ex); diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderBootstrapTest.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderBootstrapTest.java index 3bf29b698..7d851cff7 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderBootstrapTest.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderBootstrapTest.java @@ -39,6 +39,7 @@ import org.springframework.cloud.stream.binder.kafka.streams.KeyValueSerdeResolv import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.support.mapping.DefaultJackson2JavaTypeMapper; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; @@ -146,6 +147,15 @@ public class KafkaStreamsBinderBootstrapTest { .getPropertyValue("taskTopology.processorNodes[0].valDeserializer.typeResolver.arg$2"); assertThat(this.getClass().getName() + ".determineType").isEqualTo(configuredSerdeTypeResolver); + + String configuredKeyDeserializerFieldName = ((String) new DirectFieldAccessor(input2SBFB.getKafkaStreams()) + .getPropertyValue("taskTopology.processorNodes[0].keyDeserializer.typeMapper.classIdFieldName")); + assertThat(DefaultJackson2JavaTypeMapper.KEY_DEFAULT_CLASSID_FIELD_NAME).isEqualTo(configuredKeyDeserializerFieldName); + + String configuredValueDeserializerFieldName = ((String) new DirectFieldAccessor(input2SBFB.getKafkaStreams()) + .getPropertyValue("taskTopology.processorNodes[0].valDeserializer.typeMapper.classIdFieldName")); + assertThat(DefaultJackson2JavaTypeMapper.DEFAULT_CLASSID_FIELD_NAME).isEqualTo(configuredValueDeserializerFieldName); + applicationContext.close(); } @@ -164,7 +174,7 @@ public class KafkaStreamsBinderBootstrapTest { } @Bean - public Consumer>> input2() { + public Consumer, Map>> input2() { return s -> { // No-op consumer }; From df9d04fd1233c5b65c32119de8a1f82e73041452 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Tue, 11 Jan 2022 18:49:18 -0500 Subject: [PATCH 17/19] Retries for HostInfo in InteractiveQueryService InteractiveQueryService methods for finding the host info for Kafka Streams currently throw exceptions if the underlying KafkaStreams are not ready yet. Introduce a retry mechanism so that the users can control the behaviour of these methods by providing the following properties. spring.cloud.stream.kafka.streams.binder.stateStoreRetry.maxAttempts (default 1) spring.cloud.stream.kafka.streams.binder.stateStoreRetry.backoffPeriod (default 1000 ms). Resolves https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1185 --- docs/src/main/asciidoc/kafka-streams.adoc | 6 ++- .../streams/InteractiveQueryService.java | 53 +++++++++++++------ ...reamsInteractiveQueryIntegrationTests.java | 34 +++++++++++- 3 files changed, 74 insertions(+), 19 deletions(-) diff --git a/docs/src/main/asciidoc/kafka-streams.adoc b/docs/src/main/asciidoc/kafka-streams.adoc index be68b9621..1f4c3f397 100644 --- a/docs/src/main/asciidoc/kafka-streams.adoc +++ b/docs/src/main/asciidoc/kafka-streams.adoc @@ -1025,7 +1025,7 @@ ReadOnlyKeyValueStore keyValueStore = ---- During the startup, the above method call to retrieve the store might fail. -For e.g it might still be in the middle of initializing the state store. +For example, it might still be in the middle of initializing the state store. In such cases, it will be useful to retry this operation. Kafka Streams binder provides a simple retry mechanism to accommodate this. @@ -1060,6 +1060,10 @@ else { } ---- +For more information on these host finding methods, please see the Javadoc on the methods. +For these methods also, during startup, if the underlying KafkaStreams objects are not ready, they might throw exceptions. +The aforementioned retry properties are applicable for these methods as well. + ==== Other API methods available through the InteractiveQueryService Use the following API method to retrieve the `KeyQueryMetadata` object associated with the combination of given store and key. diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java index 4e35c3505..33e7890c3 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 the original author or authors. + * Copyright 2018-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -84,15 +84,7 @@ public class InteractiveQueryService { */ public T getQueryableStore(String storeName, QueryableStoreType storeType) { - RetryTemplate retryTemplate = new RetryTemplate(); - - KafkaStreamsBinderConfigurationProperties.StateStoreRetry stateStoreRetry = this.binderConfigurationProperties.getStateStoreRetry(); - RetryPolicy retryPolicy = new SimpleRetryPolicy(stateStoreRetry.getMaxAttempts()); - FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy(); - backOffPolicy.setBackOffPeriod(stateStoreRetry.getBackoffPeriod()); - - retryTemplate.setBackOffPolicy(backOffPolicy); - retryTemplate.setRetryPolicy(retryPolicy); + final RetryTemplate retryTemplate = getRetryTemplate(); KafkaStreams contextSpecificKafkaStreams = getThreadContextSpecificKafkaStreams(); @@ -191,7 +183,7 @@ public class InteractiveQueryService { * through all the consumer instances under the same application id and retrieves the * proper host. * - * Note that the end user applications must provide `applicaiton.server` as a + * Note that the end user applications must provide `application.server` as a * configuration property for all the application instances when calling this method. * If this is not available, then null maybe returned. * @param generic type for key @@ -201,11 +193,40 @@ public class InteractiveQueryService { * @return the {@link HostInfo} where the key for the provided store is hosted currently */ public HostInfo getHostInfo(String store, K key, Serializer serializer) { - final KeyQueryMetadata keyQueryMetadata = this.kafkaStreamsRegistry.getKafkaStreams() - .stream() - .map((k) -> Optional.ofNullable(k.queryMetadataForKey(store, key, serializer))) - .filter(Optional::isPresent).map(Optional::get).findFirst().orElse(null); - return keyQueryMetadata != null ? keyQueryMetadata.getActiveHost() : null; + final RetryTemplate retryTemplate = getRetryTemplate(); + + + return retryTemplate.execute(context -> { + Throwable throwable = null; + try { + final KeyQueryMetadata keyQueryMetadata = this.kafkaStreamsRegistry.getKafkaStreams() + .stream() + .map((k) -> Optional.ofNullable(k.queryMetadataForKey(store, key, serializer))) + .filter(Optional::isPresent).map(Optional::get).findFirst().orElse(null); + if (keyQueryMetadata != null) { + return keyQueryMetadata.activeHost(); + } + } + catch (Exception e) { + throwable = e; + } + throw new IllegalStateException( + "Error when retrieving state store", throwable != null ? throwable : new Throwable("Kafka Streams is not ready.")); + }); + } + + private RetryTemplate getRetryTemplate() { + RetryTemplate retryTemplate = new RetryTemplate(); + + KafkaStreamsBinderConfigurationProperties.StateStoreRetry stateStoreRetry = this.binderConfigurationProperties.getStateStoreRetry(); + RetryPolicy retryPolicy = new SimpleRetryPolicy(stateStoreRetry.getMaxAttempts()); + FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy(); + backOffPolicy.setBackOffPeriod(stateStoreRetry.getBackoffPeriod()); + + retryTemplate.setBackOffPolicy(backOffPolicy); + retryTemplate.setRetryPolicy(retryPolicy); + + return retryTemplate; } /** diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java index 83ab43ef0..7acd4182f 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2021 the original author or authors. + * Copyright 2017-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,6 +26,7 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.serialization.IntegerSerializer; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyQueryMetadata; import org.apache.kafka.streams.KeyValue; @@ -126,6 +127,35 @@ public class KafkaStreamsInteractiveQueryIntegrationTests { .store(StoreQueryParameters.fromNameAndType("foo", storeType)); } + @Test + public void testStateStoreRetrievalRetryForHostInfoService() { + StreamsBuilderFactoryBean mock = Mockito.mock(StreamsBuilderFactoryBean.class); + KafkaStreams mockKafkaStreams = Mockito.mock(KafkaStreams.class); + Mockito.when(mock.getKafkaStreams()).thenReturn(mockKafkaStreams); + KafkaStreamsRegistry kafkaStreamsRegistry = new KafkaStreamsRegistry(); + kafkaStreamsRegistry.registerKafkaStreams(mock); + Mockito.when(mock.isRunning()).thenReturn(true); + Properties mockProperties = new Properties(); + mockProperties.put(StreamsConfig.APPLICATION_ID_CONFIG, "foobarApp-123"); + Mockito.when(mock.getStreamsConfiguration()).thenReturn(mockProperties); + KafkaStreamsBinderConfigurationProperties binderConfigurationProperties = + new KafkaStreamsBinderConfigurationProperties(new KafkaProperties()); + binderConfigurationProperties.getStateStoreRetry().setMaxAttempts(3); + InteractiveQueryService interactiveQueryService = new InteractiveQueryService(kafkaStreamsRegistry, + binderConfigurationProperties); + + QueryableStoreType> storeType = QueryableStoreTypes.keyValueStore(); + final StringSerializer serializer = new StringSerializer(); + try { + interactiveQueryService.getHostInfo("foo", "fooKey", serializer); + } + catch (Exception ignored) { + + } + Mockito.verify(mockKafkaStreams, times(3)) + .queryMetadataForKey("foo", "fooKey", serializer); + } + @Test public void testKstreamBinderWithPojoInputAndStringOuput() { SpringApplication app = new SpringApplication(ProductCountApplication.class); @@ -264,4 +294,4 @@ public class KafkaStreamsInteractiveQueryIntegrationTests { } -} +} \ No newline at end of file From 31b91f47e43626aafa7aff25d0efa1071da74e68 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Wed, 12 Jan 2022 12:23:22 -0500 Subject: [PATCH 18/19] Fixing InteractiveQueryService test. Fixing checkstyle issues. --- .../kafka/streams/InteractiveQueryService.java | 2 +- ...kaStreamsInteractiveQueryIntegrationTests.java | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java index 33e7890c3..d8f350af6 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java @@ -211,7 +211,7 @@ public class InteractiveQueryService { throwable = e; } throw new IllegalStateException( - "Error when retrieving state store", throwable != null ? throwable : new Throwable("Kafka Streams is not ready.")); + "Error when retrieving state store.", throwable != null ? throwable : new Throwable("Kafka Streams is not ready.")); }); } diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java index 7acd4182f..a924c2d2e 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java @@ -63,6 +63,7 @@ import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.internal.verification.VerificationModeFactory.times; /** @@ -147,13 +148,13 @@ public class KafkaStreamsInteractiveQueryIntegrationTests { QueryableStoreType> storeType = QueryableStoreTypes.keyValueStore(); final StringSerializer serializer = new StringSerializer(); try { - interactiveQueryService.getHostInfo("foo", "fooKey", serializer); + interactiveQueryService.getHostInfo("foo", "foobarApp-key", serializer); } catch (Exception ignored) { } Mockito.verify(mockKafkaStreams, times(3)) - .queryMetadataForKey("foo", "fooKey", serializer); + .queryMetadataForKey("foo", "foobarApp-key", serializer); } @Test @@ -224,16 +225,16 @@ public class KafkaStreamsInteractiveQueryIntegrationTests { assertThat(hostInfo.host() + ":" + hostInfo.port()) .isEqualTo(embeddedKafka.getBrokersAsString()); - HostInfo hostInfoFoo = interactiveQueryService - .getHostInfo("prod-id-count-store-foo", 123, new IntegerSerializer()); - assertThat(hostInfoFoo).isNull(); + assertThatThrownBy(() -> interactiveQueryService + .getHostInfo("prod-id-count-store-foo", 123, new IntegerSerializer())) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Error when retrieving state store."); final List hostInfos = interactiveQueryService.getAllHostsInfo("prod-id-count-store"); assertThat(hostInfos.size()).isEqualTo(1); final HostInfo hostInfo1 = hostInfos.get(0); assertThat(hostInfo1.host() + ":" + hostInfo1.port()) .isEqualTo(embeddedKafka.getBrokersAsString()); - } @EnableAutoConfiguration @@ -294,4 +295,4 @@ public class KafkaStreamsInteractiveQueryIntegrationTests { } -} \ No newline at end of file +} From d345ac88b1a3682f3e1297da9ced1dfc1b2e45eb Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Wed, 12 Jan 2022 17:12:01 -0500 Subject: [PATCH 19/19] Enable custom binder health check impelementation Currently, KafkaBinderHealthIndicator is not customizable and included by default when Spring Boot actuator is on the classpath. Fix this by allowing the application to provide a custom implementation. A new marker interface called KafkaBinderHealth can be used by the applicaiton to provide a custom HealthIndicator implementation, in which case, the binder's default implementation will be excluded. Tests and docs changes. Resolves https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1180 --- docs/src/main/asciidoc/overview.adoc | 25 +++++++ .../binder/kafka/KafkaBinderHealth.java | 29 ++++++++ .../kafka/KafkaBinderHealthIndicator.java | 9 ++- ...fkaBinderHealthIndicatorConfiguration.java | 8 ++- .../KafkaBinderCustomHealthCheckTests.java | 72 +++++++++++++++++++ 5 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealth.java create mode 100644 spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/KafkaBinderCustomHealthCheckTests.java diff --git a/docs/src/main/asciidoc/overview.adoc b/docs/src/main/asciidoc/overview.adoc index b3dca2399..367ae107c 100644 --- a/docs/src/main/asciidoc/overview.adoc +++ b/docs/src/main/asciidoc/overview.adoc @@ -969,3 +969,28 @@ public AdminClientConfigCustomizer adminClientConfigCustomizer() { }; } ``` + +[[custom-kafka-binder-health-indicator]] +=== Custom Kafka Binder Health Indicator + +Kafka binder activates a default health indicator when Spring Boot actuator is on the classpath. +This health indicator checks the health of the binder and any communication issues with the Kafka broker. +If an application wants to disable this default health check implementation and include a custom implementation, then it can provide an implementation for `KafkaBinderHealth` interface. +`KafkaBinderHealth` is a marker interface that extends from `HealthIndicator`. +In the custom implementation, it must provide an implementation for the `health()` method. +The custom implementation must be present in the application configuration as a bean. +When the binder discovers the custom implementation, it will use that instead of the default implementation. +Here is an example of such a custom implementation bean in the application. + +``` +@Bean +public KafkaBinderHealth kafkaBinderHealthIndicator() { + return new KafkaBinderHealth() { + @Override + public Health health() { + // custom implementation details. + } + }; +} +``` + diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealth.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealth.java new file mode 100644 index 000000000..dd6e6b6b1 --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealth.java @@ -0,0 +1,29 @@ +/* + * Copyright 2022-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import org.springframework.boot.actuate.health.HealthIndicator; + +/** + * Marker interface used for custom KafkaBinderHealth indicator implementations. + * + * @author Soby Chacko + * @since 3.2.2 + */ +public interface KafkaBinderHealth extends HealthIndicator { + +} diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicator.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicator.java index 594ca55cc..e35d15290 100644 --- a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicator.java +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicator.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2021 the original author or authors. + * Copyright 2016-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,6 @@ import org.apache.kafka.common.PartitionInfo; import org.springframework.beans.factory.DisposableBean; import org.springframework.boot.actuate.health.Health; -import org.springframework.boot.actuate.health.HealthIndicator; import org.springframework.boot.actuate.health.Status; import org.springframework.boot.actuate.health.StatusAggregator; import org.springframework.kafka.core.ConsumerFactory; @@ -55,7 +54,7 @@ import org.springframework.scheduling.concurrent.CustomizableThreadFactory; * @author Chukwubuikem Ume-Ugwa * @author Taras Danylchuk */ -public class KafkaBinderHealthIndicator implements HealthIndicator, DisposableBean { +public class KafkaBinderHealthIndicator implements KafkaBinderHealth, DisposableBean { private static final int DEFAULT_TIMEOUT = 60; @@ -73,7 +72,7 @@ public class KafkaBinderHealthIndicator implements HealthIndicator, DisposableBe private boolean considerDownWhenAnyPartitionHasNoLeader; public KafkaBinderHealthIndicator(KafkaMessageChannelBinder binder, - ConsumerFactory consumerFactory) { + ConsumerFactory consumerFactory) { this.binder = binder; this.consumerFactory = consumerFactory; } @@ -219,7 +218,7 @@ public class KafkaBinderHealthIndicator implements HealthIndicator, DisposableBe } @Override - public void destroy() throws Exception { + public void destroy() { executor.shutdown(); } diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderHealthIndicatorConfiguration.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderHealthIndicatorConfiguration.java index dd2dc5702..67d53a182 100644 --- a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderHealthIndicatorConfiguration.java +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderHealthIndicatorConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 the original author or authors. + * Copyright 2018-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,8 @@ import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.springframework.boot.actuate.autoconfigure.health.ConditionalOnEnabledHealthIndicator; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealth; import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealthIndicator; import org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder; import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; @@ -38,11 +40,13 @@ import org.springframework.util.ObjectUtils; * * @author Oleg Zhurakousky * @author Chukwubuikem Ume-Ugwa + * @author Soby Chacko */ -@Configuration +@Configuration(proxyBeanMethods = false) @ConditionalOnClass(name = "org.springframework.boot.actuate.health.HealthIndicator") @ConditionalOnEnabledHealthIndicator("binders") +@ConditionalOnMissingBean(KafkaBinderHealth.class) public class KafkaBinderHealthIndicatorConfiguration { @Bean diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/KafkaBinderCustomHealthCheckTests.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/KafkaBinderCustomHealthCheckTests.java new file mode 100644 index 000000000..49a6c1707 --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/KafkaBinderCustomHealthCheckTests.java @@ -0,0 +1,72 @@ +/* + * Copyright 2022-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.integration2; + +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.beans.factory.NoSuchBeanDefinitionException; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealth; +import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealthIndicator; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; + +/** + * @author Soby Chacko + */ +public class KafkaBinderCustomHealthCheckTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, 10); + + @Test + public void testCustomHealthIndicatorIsActivated() { + ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder( + CustomHealthCheckApplication.class).web(WebApplicationType.NONE).run( + "--spring.cloud.stream.kafka.binder.brokers=" + + embeddedKafka.getEmbeddedKafka().getBrokersAsString()); + final KafkaBinderHealth kafkaBinderHealth = applicationContext.getBean(KafkaBinderHealth.class); + assertThat(kafkaBinderHealth).isInstanceOf(CustomHealthIndicator.class); + assertThatThrownBy(() -> applicationContext.getBean(KafkaBinderHealthIndicator.class)).isInstanceOf(NoSuchBeanDefinitionException.class); + applicationContext.close(); + } + + @SpringBootApplication + static class CustomHealthCheckApplication { + + @Bean + public CustomHealthIndicator kafkaBinderHealthIndicator() { + return new CustomHealthIndicator(); + } + } + + static class CustomHealthIndicator implements KafkaBinderHealth { + + @Override + public Health health() { + return null; + } + } +}