diff --git a/docs/pom.xml b/docs/pom.xml index 6552c0ae0..4c5528330 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -7,7 +7,7 @@ org.springframework.cloud spring-cloud-stream-binder-kafka-parent - 3.2.2-SNAPSHOT + 4.0.0-SNAPSHOT jar spring-cloud-stream-binder-kafka-docs diff --git a/docs/src/main/asciidoc/kafka-streams.adoc b/docs/src/main/asciidoc/kafka-streams.adoc index 0052531b3..1f4c3f397 100644 --- a/docs/src/main/asciidoc/kafka-streams.adoc +++ b/docs/src/main/asciidoc/kafka-streams.adoc @@ -430,213 +430,6 @@ public Function, KStream> bar() { You can compose them as `foo|bar`, but keep in mind that the second function (`bar` in this case) must have a `KTable` as input since the first function (`foo`) has `KTable` as output. -==== Imperative programming model. - -Starting with `3.1.0` version of the binder, we recommend using the functional programming model described above for Kafka Streams binder based applications. -The support for `StreamListener` is deprecated starting with `3.1.0` of Spring Cloud Stream. -Below, we are providing some details on the `StreamListener` based Kafka Streams processors as a reference. - -Following is the equivalent of the Word count example using `StreamListener`. - -[source] ----- -@SpringBootApplication -@EnableBinding(KafkaStreamsProcessor.class) -public class WordCountProcessorApplication { - - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - return input - .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) - .groupBy((key, value) -> value) - .windowedBy(TimeWindows.of(5000)) - .count(Materialized.as("WordCounts-multi")) - .toStream() - .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))); - } - - public static void main(String[] args) { - SpringApplication.run(WordCountProcessorApplication.class, args); - } ----- - -As you can see, this is a bit more verbose since you need to provide `EnableBinding` and the other extra annotations like `StreamListener` and `SendTo` to make it a complete application. -`EnableBinding` is where you specify your binding interface that contains your bindings. -In this case, we are using the stock `KafkaStreamsProcessor` binding interface that has the following contracts. - -[source] ----- -public interface KafkaStreamsProcessor { - - @Input("input") - KStream input(); - - @Output("output") - KStream output(); - -} ----- - -Binder will create bindings for the input `KStream` and output `KStream` since you are using a binding interface that contains those declarations. - -In addition to the obvious differences in the programming model offered in the functional style, one particular thing that needs to be mentioned here is that the binding names are what you specify in the binding interface. -For example, in the above application, since we are using `KafkaStreamsProcessor`, the binding names are `input` and `output`. -Binding properties need to use those names. For instance `spring.cloud.stream.bindings.input.destination`, `spring.cloud.stream.bindings.output.destination` etc. -Keep in mind that this is fundamentally different from the functional style since there the binder generates binding names for the application. -This is because the application does not provide any binding interfaces in the functional model using `EnableBinding`. - -Here is another example of a sink where we have two inputs. - -[source] ----- -@EnableBinding(KStreamKTableBinding.class) -..... -..... -@StreamListener -public void process(@Input("inputStream") KStream playEvents, - @Input("inputTable") KTable songTable) { - .... - .... -} - -interface KStreamKTableBinding { - - @Input("inputStream") - KStream inputStream(); - - @Input("inputTable") - KTable inputTable(); -} - ----- - -Following is the `StreamListener` equivalent of the same `BiFunction` based processor that we saw above. - - -[source] ----- -@EnableBinding(KStreamKTableBinding.class) -.... -.... - -@StreamListener -@SendTo("output") -public KStream process(@Input("input") KStream userClicksStream, - @Input("inputTable") KTable userRegionsTable) { -.... -.... -} - -interface KStreamKTableBinding extends KafkaStreamsProcessor { - - @Input("inputX") - KTable inputTable(); -} ----- - -Finally, here is the `StreamListener` equivalent of the application with three inputs and curried functions. - -[source] ----- -@EnableBinding(CustomGlobalKTableProcessor.class) -... -... - @StreamListener - @SendTo("output") - public KStream process( - @Input("input-1") KStream ordersStream, - @Input("input-2") GlobalKTable customers, - @Input("input-3") GlobalKTable products) { - - KStream customerOrdersStream = ordersStream.join( - customers, (orderId, order) -> order.getCustomerId(), - (order, customer) -> new CustomerOrder(customer, order)); - - return customerOrdersStream.join(products, - (orderId, customerOrder) -> customerOrder.productId(), - (customerOrder, product) -> { - EnrichedOrder enrichedOrder = new EnrichedOrder(); - enrichedOrder.setProduct(product); - enrichedOrder.setCustomer(customerOrder.customer); - enrichedOrder.setOrder(customerOrder.order); - return enrichedOrder; - }); - } - - interface CustomGlobalKTableProcessor { - - @Input("input-1") - KStream input1(); - - @Input("input-2") - GlobalKTable input2(); - - @Input("input-3") - GlobalKTable input3(); - - @Output("output") - KStream output(); - } - ----- - -You might notice that the above two examples are even more verbose since in addition to provide `EnableBinding`, you also need to write your own custom binding interface as well. -Using the functional model, you can avoid all those ceremonial details. - -Before we move on from looking at the general programming model offered by Kafka Streams binder, here is the `StreamListener` version of multiple output bindings. - -[source] ----- -EnableBinding(KStreamProcessorWithBranches.class) -public static class WordCountProcessorApplication { - - @Autowired - private TimeWindows timeWindows; - - @StreamListener("input") - @SendTo({"output1","output2","output3"}) - public KStream[] process(KStream input) { - - Predicate isEnglish = (k, v) -> v.word.equals("english"); - Predicate isFrench = (k, v) -> v.word.equals("french"); - Predicate isSpanish = (k, v) -> v.word.equals("spanish"); - - return input - .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) - .groupBy((key, value) -> value) - .windowedBy(timeWindows) - .count(Materialized.as("WordCounts-1")) - .toStream() - .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))) - .branch(isEnglish, isFrench, isSpanish); - } - - interface KStreamProcessorWithBranches { - - @Input("input") - KStream input(); - - @Output("output1") - KStream output1(); - - @Output("output2") - KStream output2(); - - @Output("output3") - KStream output3(); - } -} ----- - -To recap, we have reviewed the various programming model choices when using the Kafka Streams binder. - -The binder provides binding capabilities for `KStream`, `KTable` and `GlobalKTable` on the input. -`KTable` and `GlobalKTable` bindings are only available on the input. -Binder supports both input and output bindings for `KStream`. - -The upshot of the programming model of Kafka Streams binder is that the binder provides you the flexibility of going with a fully functional programming model or using the `StreamListener` based imperative approach. - === Ancillaries to the programming model ==== Multiple Kafka Streams processors within a single application @@ -677,7 +470,7 @@ This is also true when you have a single Kafka Streams processor and other types Application id is a mandatory property that you need to provide for a Kafka Streams application. Spring Cloud Stream Kafka Streams binder allows you to configure this application id in multiple ways. -If you only have one single processor or `StreamListener` in the application, then you can set this at the binder level using the following property: +If you only have one single processor in the application, then you can set this at the binder level using the following property: `spring.cloud.stream.kafka.streams.binder.applicationId`. @@ -712,33 +505,6 @@ and `spring.cloud.stream.kafka.streams.binder.functions.anotherProcess.applicationId` -In the case of `StreamListener`, you need to set this on the first input binding on the processor. - -For e.g. imagine that you have the following two `StreamListener` based processors. - -``` -@StreamListener -@SendTo("output") -public KStream process(@Input("input") > input) { - ... -} - -@StreamListener -@SendTo("anotherOutput") -public KStream anotherProcess(@Input("anotherInput") > input) { - ... -} -``` - -Then you must set the application id for this using the following binding property. - -`spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId` - -and - -`spring.cloud.stream.kafka.streams.bindings.anotherInput.consumer.applicationId` - - For function based model also, this approach of setting application id at the binding level will work. However, setting per function at the binder level as we have seen above is much easier if you are using the functional model. @@ -749,14 +515,12 @@ If the application does not provide an application ID, then in that case the bin This is convenient in development scenarios as it avoids the need for explicitly providing the application ID. The generated application ID in this manner will be static over application restarts. In the case of functional model, the generated application ID will be the function bean name followed by the literal `applicationID`, for e.g `process-applicationID` if `process` if the function bean name. -In the case of `StreamListener`, instead of using the function bean name, the generated application ID will be use the containing class name followed by the method name followed by the literal `applicationId`. ====== Summary of setting Application ID -* By default, binder will auto generate the application ID per function or `StreamListener` methods. +* By default, binder will auto generate the application ID per function methods. * If you have a single processor, then you can use `spring.kafka.streams.applicationId`, `spring.application.name` or `spring.cloud.stream.kafka.streams.binder.applicationId`. * If you have multiple processors, then application ID can be set per function using the property - `spring.cloud.stream.kafka.streams.binder.functions..applicationId`. -In the case of `StreamListener`, this can be done using `spring.cloud.stream.kafka.streams.bindings.input.applicationId`, assuming that the input binding name is `input`. ==== Overriding the default binding names generated by the binder with the functional style @@ -816,7 +580,7 @@ Keys are always deserialized using native Serdes. For values, by default, deserialization on the inbound is natively performed by Kafka. Please note that this is a major change on default behavior from previous versions of Kafka Streams binder where the deserialization was done by the framework. -Kafka Streams binder will try to infer matching `Serde` types by looking at the type signature of `java.util.function.Function|Consumer` or `StreamListener`. +Kafka Streams binder will try to infer matching `Serde` types by looking at the type signature of `java.util.function.Function|Consumer`. Here is the order that it matches the Serdes. * If the application provides a bean of type `Serde` and if the return type is parameterized with the actual type of the incoming key or value type, then it will use that `Serde` for inbound deserialization. @@ -1016,7 +780,7 @@ It is always recommended to explicitly create a DLQ topic for each input binding ==== DLQ per input consumer binding The property `spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler` is applicable for the entire application. -This implies that if there are multiple functions or `StreamListener` methods in the same application, this property is applied to all of them. +This implies that if there are multiple functions in the same application, this property is applied to all of them. However, if you have multiple processors or multiple input bindings within a single processor, then you can use the finer-grained DLQ control that the binder provides per input consumer binding. If you have the following processor, @@ -1061,7 +825,7 @@ If you set a consumer binding's `dlqPartitions` property to a value greater than A couple of things to keep in mind when using the exception handling feature in Kafka Streams binder. * The property `spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler` is applicable for the entire application. -This implies that if there are multiple functions or `StreamListener` methods in the same application, this property is applied to all of them. +This implies that if there are multiple functions in the same application, this property is applied to all of them. * The exception handling for deserialization works consistently with native deserialization and framework provided message conversion. ==== Handling Production Exceptions in the Binder @@ -2104,7 +1868,7 @@ Default: `logAndFail` applicationId:: Convenient way to set the application.id for the Kafka Streams application globally at the binder level. -If the application contains multiple functions or `StreamListener` methods, then the application id should be set differently. +If the application contains multiple functions, then the application id should be set differently. See above where setting the application id is discussed in detail. + Default: application will generate a static application ID. See the application ID section for more details. @@ -2168,7 +1932,7 @@ The following properties are available for Kafka Streams consumers and must be p For convenience, if there are multiple input bindings and they all require a common value, that can be configured by using the prefix `spring.cloud.stream.kafka.streams.default.consumer.`. applicationId:: -Setting application.id per input binding. This is only preferred for `StreamListener` based processors, for function based processors see other approaches outlined above. +Setting application.id per input binding. + Default: See above. @@ -2242,7 +2006,7 @@ In Kafka Streams, you can control of the number of threads a processor can creat This, you can do using the various `configuration` options described above under binder, functions, producer or consumer level. You can also use the `concurrency` property that core Spring Cloud Stream provides for this purpose. When using this, you need to use it on the consumer. -When you have more than one input bindings either in a function or `StreamListener`, set this on the first input binding. +When you have more than one input binding, set this on the first input binding. For e.g. when setting `spring.cloud.stream.bindings.process-in-0.consumer.concurrency`, it will be translated as `num.stream.threads` by the binder. If you have multiple processors and one processor defines binding level concurrency, but not the others, those ones with no binding level concurrency will default back to the binder wide property specified through `spring.cloud.stream.kafka.streams.binder.configuration.num.stream.threads`. diff --git a/docs/src/main/asciidoc/overview.adoc b/docs/src/main/asciidoc/overview.adoc index 7261c46f5..367ae107c 100644 --- a/docs/src/main/asciidoc/overview.adoc +++ b/docs/src/main/asciidoc/overview.adoc @@ -364,8 +364,6 @@ Starting with version 3.0, when `spring.cloud.stream.binding..consumer.bat Otherwise, the method will be called with one record at a time. The size of the batch is controlled by Kafka consumer properties `max.poll.records`, `fetch.min.bytes`, `fetch.max.wait.ms`; refer to the Kafka documentation for more information. -Bear in mind that batch mode is not supported with `@StreamListener` - it only works with the newer functional programming model. - IMPORTANT: Retry within the binder is not supported when using batch mode, so `maxAttempts` will be overridden to 1. You can configure a `SeekToCurrentBatchErrorHandler` (using a `ListenerContainerCustomizer`) to achieve similar functionality to retry in the binder. You can also use a manual `AckMode` and call `Ackowledgment.nack(index, sleep)` to commit the offsets for a partial batch and have the remaining records redelivered. diff --git a/pom.xml b/pom.xml index 00a2fe700..cb45e73dc 100644 --- a/pom.xml +++ b/pom.xml @@ -2,12 +2,12 @@ 4.0.0 spring-cloud-stream-binder-kafka-parent - 3.2.2-SNAPSHOT + 4.0.0-SNAPSHOT pom org.springframework.cloud spring-cloud-build - 3.1.0 + 4.0.0-SNAPSHOT @@ -20,11 +20,11 @@ HEAD - 1.8 - 2.8.0 - 5.5.5 + 17 + 3.0.0-SNAPSHOT + 6.0.0-SNAPSHOT 3.0.0 - 3.2.2-SNAPSHOT + 4.0.0-SNAPSHOT true true true @@ -144,7 +144,7 @@ org.apache.maven.plugins maven-antrun-plugin - 1.7 + org.apache.maven.plugins diff --git a/spring-cloud-starter-stream-kafka/pom.xml b/spring-cloud-starter-stream-kafka/pom.xml index 23b336833..0959fbc20 100644 --- a/spring-cloud-starter-stream-kafka/pom.xml +++ b/spring-cloud-starter-stream-kafka/pom.xml @@ -4,7 +4,7 @@ org.springframework.cloud spring-cloud-stream-binder-kafka-parent - 3.2.2-SNAPSHOT + 4.0.0-SNAPSHOT spring-cloud-starter-stream-kafka Spring Cloud Starter Stream Kafka diff --git a/spring-cloud-stream-binder-kafka-core/pom.xml b/spring-cloud-stream-binder-kafka-core/pom.xml index af869b9c6..51330e69b 100644 --- a/spring-cloud-stream-binder-kafka-core/pom.xml +++ b/spring-cloud-stream-binder-kafka-core/pom.xml @@ -5,7 +5,7 @@ org.springframework.cloud spring-cloud-stream-binder-kafka-parent - 3.2.2-SNAPSHOT + 4.0.0-SNAPSHOT spring-cloud-stream-binder-kafka-core Spring Cloud Stream Kafka Binder Core diff --git a/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java b/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java index c53566eef..b8ee4d351 100644 --- a/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java +++ b/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java @@ -28,10 +28,9 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import javax.validation.constraints.AssertTrue; -import javax.validation.constraints.Min; -import javax.validation.constraints.NotNull; - +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.kafka.clients.consumer.ConsumerConfig; diff --git a/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaProducerProperties.java b/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaProducerProperties.java index 3653c5f61..54ca36713 100644 --- a/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaProducerProperties.java +++ b/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaProducerProperties.java @@ -19,7 +19,7 @@ package org.springframework.cloud.stream.binder.kafka.properties; import java.util.HashMap; import java.util.Map; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; import org.springframework.expression.Expression; diff --git a/spring-cloud-stream-binder-kafka-streams/pom.xml b/spring-cloud-stream-binder-kafka-streams/pom.xml index 36c7f5bac..24e93421d 100644 --- a/spring-cloud-stream-binder-kafka-streams/pom.xml +++ b/spring-cloud-stream-binder-kafka-streams/pom.xml @@ -10,7 +10,7 @@ org.springframework.cloud spring-cloud-stream-binder-kafka-parent - 3.2.2-SNAPSHOT + 4.0.0-SNAPSHOT @@ -73,35 +73,6 @@ kafka_2.13 test - - - org.apache.avro - avro - ${avro.version} - provided - - - - - org.apache.avro - avro-maven-plugin - ${avro.version} - - - generate-test-sources - - schema - - - ${project.basedir}/target/generated-test-sources - ${project.basedir}/target/generated-test-sources - ${project.basedir}/src/test/resources/avro - - - - - - diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerParameterAdapter.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerParameterAdapter.java deleted file mode 100644 index d2645c957..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerParameterAdapter.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2017-2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams; - -import org.apache.kafka.streams.kstream.KStream; - -import org.springframework.cloud.stream.binding.StreamListenerParameterAdapter; -import org.springframework.core.MethodParameter; -import org.springframework.core.ResolvableType; - -/** - * {@link StreamListenerParameterAdapter} for KStream. - * - * @author Marius Bogoevici - * @author Soby Chacko - */ -class KStreamStreamListenerParameterAdapter - implements StreamListenerParameterAdapter, KStream> { - - private final KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate; - - private final KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue; - - KStreamStreamListenerParameterAdapter( - KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate, - KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue) { - this.kafkaStreamsMessageConversionDelegate = kafkaStreamsMessageConversionDelegate; - this.KafkaStreamsBindingInformationCatalogue = KafkaStreamsBindingInformationCatalogue; - } - - @Override - public boolean supports(Class bindingTargetType, MethodParameter methodParameter) { - return KafkaStreamsBinderUtils.supportsKStream(methodParameter, bindingTargetType); - } - - @Override - @SuppressWarnings("unchecked") - public KStream adapt(KStream bindingTarget, MethodParameter parameter) { - ResolvableType resolvableType = ResolvableType.forMethodParameter(parameter); - final Class valueClass = (resolvableType.getGeneric(1).getRawClass() != null) - ? (resolvableType.getGeneric(1).getRawClass()) : Object.class; - if (this.KafkaStreamsBindingInformationCatalogue - .isUseNativeDecoding(bindingTarget)) { - return bindingTarget; - } - else { - return this.kafkaStreamsMessageConversionDelegate - .deserializeOnInbound(valueClass, bindingTarget); - } - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerResultAdapter.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerResultAdapter.java deleted file mode 100644 index 6ffce5732..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerResultAdapter.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2017-2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams; - -import java.io.Closeable; -import java.io.IOException; - -import org.apache.kafka.streams.kstream.KStream; - -import org.springframework.cloud.stream.binding.StreamListenerResultAdapter; - -/** - * {@link StreamListenerResultAdapter} for KStream. - * - * @author Marius Bogoevici - * @author Soby Chacko - */ -class KStreamStreamListenerResultAdapter implements - StreamListenerResultAdapter { - - @Override - public boolean supports(Class resultType, Class boundElement) { - return KStream.class.isAssignableFrom(resultType) - && KStream.class.isAssignableFrom(boundElement); - } - - @Override - @SuppressWarnings("unchecked") - public Closeable adapt(KStream streamListenerResult, - KStreamBoundElementFactory.KStreamWrapper boundElement) { - boundElement.wrap(streamListenerResult); - return new NoOpCloseable(); - } - - private static final class NoOpCloseable implements Closeable { - - @Override - public void close() throws IOException { - - } - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderSupportAutoConfiguration.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderSupportAutoConfiguration.java index 17c4d8723..e522988b8 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderSupportAutoConfiguration.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderSupportAutoConfiguration.java @@ -17,7 +17,6 @@ package org.springframework.cloud.stream.binder.kafka.streams; import java.lang.reflect.Constructor; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,10 +49,7 @@ import org.springframework.cloud.stream.binder.BinderConfiguration; import org.springframework.cloud.stream.binder.kafka.streams.function.FunctionDetectorCondition; import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; -import org.springframework.cloud.stream.binder.kafka.streams.serde.CompositeNonNativeSerde; -import org.springframework.cloud.stream.binder.kafka.streams.serde.MessageConverterDelegateSerde; import org.springframework.cloud.stream.binding.BindingService; -import org.springframework.cloud.stream.binding.StreamListenerResultAdapter; import org.springframework.cloud.stream.config.BinderProperties; import org.springframework.cloud.stream.config.BindingServiceConfiguration; import org.springframework.cloud.stream.config.BindingServiceProperties; @@ -296,37 +292,6 @@ public class KafkaStreamsBinderSupportAutoConfiguration { } } - @Bean - public KStreamStreamListenerResultAdapter kstreamStreamListenerResultAdapter() { - return new KStreamStreamListenerResultAdapter(); - } - - @Bean - public KStreamStreamListenerParameterAdapter kstreamStreamListenerParameterAdapter( - KafkaStreamsMessageConversionDelegate kstreamBoundMessageConversionDelegate, - KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue) { - return new KStreamStreamListenerParameterAdapter( - kstreamBoundMessageConversionDelegate, - KafkaStreamsBindingInformationCatalogue); - } - - @Bean - public KafkaStreamsStreamListenerSetupMethodOrchestrator kafkaStreamsStreamListenerSetupMethodOrchestrator( - BindingServiceProperties bindingServiceProperties, - KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties, - KeyValueSerdeResolver keyValueSerdeResolver, - KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, - KStreamStreamListenerParameterAdapter kafkaStreamListenerParameterAdapter, - Collection streamListenerResultAdapters, - ObjectProvider cleanupConfig, - ObjectProvider customizerProvider, ConfigurableEnvironment environment) { - return new KafkaStreamsStreamListenerSetupMethodOrchestrator( - bindingServiceProperties, kafkaStreamsExtendedBindingProperties, - keyValueSerdeResolver, kafkaStreamsBindingInformationCatalogue, - kafkaStreamListenerParameterAdapter, streamListenerResultAdapters, - cleanupConfig.getIfUnique(), customizerProvider.getIfUnique(), environment); - } - @Bean public KafkaStreamsMessageConversionDelegate messageConversionDelegate( @Qualifier(IntegrationContextUtils.ARGUMENT_RESOLVER_MESSAGE_CONVERTER_BEAN_NAME) @@ -338,20 +303,6 @@ public class KafkaStreamsBinderSupportAutoConfiguration { KafkaStreamsBindingInformationCatalogue, binderConfigurationProperties); } - @Bean - public MessageConverterDelegateSerde messageConverterDelegateSerde( - @Qualifier(IntegrationContextUtils.ARGUMENT_RESOLVER_MESSAGE_CONVERTER_BEAN_NAME) - CompositeMessageConverter compositeMessageConverterFactory) { - return new MessageConverterDelegateSerde(compositeMessageConverterFactory); - } - - @Bean - public CompositeNonNativeSerde compositeNonNativeSerde( - @Qualifier(IntegrationContextUtils.ARGUMENT_RESOLVER_MESSAGE_CONVERTER_BEAN_NAME) - CompositeMessageConverter compositeMessageConverterFactory) { - return new CompositeNonNativeSerde(compositeMessageConverterFactory); - } - @Bean public KStreamBoundElementFactory kStreamBoundElementFactory( BindingServiceProperties bindingServiceProperties, diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java index 366da7e04..92b856d6b 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java @@ -42,7 +42,7 @@ import org.springframework.util.CollectionUtils; * A catalogue that provides binding information for Kafka Streams target types such as * KStream. It also keeps a catalogue for the underlying {@link StreamsBuilderFactoryBean} * and {@link StreamsConfig} associated with various - * {@link org.springframework.cloud.stream.annotation.StreamListener} methods in the + * Kafka Streams functions in the * {@link org.springframework.context.ApplicationContext}. * * @author Soby Chacko diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionProcessor.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionProcessor.java index c91953715..ff0286c1a 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionProcessor.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionProcessor.java @@ -51,7 +51,6 @@ import org.springframework.cloud.stream.binder.kafka.streams.function.KafkaStrea import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; -import org.springframework.cloud.stream.binding.StreamListenerErrorMessages; import org.springframework.cloud.stream.config.BindingProperties; import org.springframework.cloud.stream.config.BindingServiceProperties; import org.springframework.cloud.stream.function.FunctionConstants; @@ -562,7 +561,7 @@ public class KafkaStreamsFunctionProcessor extends AbstractKafkaStreamsBinderPro } } else { - throw new IllegalStateException(StreamListenerErrorMessages.INVALID_DECLARATIVE_METHOD_PARAMETERS); + //throw new IllegalStateException(StreamListenerErrorMessages.INVALID_DECLARATIVE_METHOD_PARAMETERS); } } return arguments; diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsStreamListenerSetupMethodOrchestrator.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsStreamListenerSetupMethodOrchestrator.java deleted file mode 100644 index 6e315faf1..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsStreamListenerSetupMethodOrchestrator.java +++ /dev/null @@ -1,521 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams; - -import java.lang.reflect.Method; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.streams.StreamsBuilder; -import org.apache.kafka.streams.StreamsConfig; -import org.apache.kafka.streams.Topology; -import org.apache.kafka.streams.kstream.GlobalKTable; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.KTable; -import org.apache.kafka.streams.state.StoreBuilder; -import org.apache.kafka.streams.state.Stores; - -import org.springframework.beans.factory.BeanInitializationException; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsStateStore; -import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; -import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; -import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsStateStoreProperties; -import org.springframework.cloud.stream.binding.StreamListenerErrorMessages; -import org.springframework.cloud.stream.binding.StreamListenerParameterAdapter; -import org.springframework.cloud.stream.binding.StreamListenerResultAdapter; -import org.springframework.cloud.stream.binding.StreamListenerSetupMethodOrchestrator; -import org.springframework.cloud.stream.config.BindingProperties; -import org.springframework.cloud.stream.config.BindingServiceProperties; -import org.springframework.context.ApplicationContext; -import org.springframework.core.MethodParameter; -import org.springframework.core.ResolvableType; -import org.springframework.core.annotation.AnnotationUtils; -import org.springframework.core.env.ConfigurableEnvironment; -import org.springframework.kafka.config.StreamsBuilderFactoryBean; -import org.springframework.kafka.config.StreamsBuilderFactoryBeanConfigurer; -import org.springframework.kafka.core.CleanupConfig; -import org.springframework.messaging.handler.annotation.SendTo; -import org.springframework.util.Assert; -import org.springframework.util.ObjectUtils; -import org.springframework.util.ReflectionUtils; -import org.springframework.util.StringUtils; - -/** - * Kafka Streams specific implementation for {@link StreamListenerSetupMethodOrchestrator} - * that overrides the default mechanisms for invoking StreamListener adapters. - *

- * The orchestration primarily focus on the following areas: - *

- * 1. Allow multiple KStream output bindings (KStream branching) by allowing more than one - * output values on {@link SendTo} 2. Allow multiple inbound bindings for multiple KStream - * and or KTable/GlobalKTable types. 3. Each StreamListener method that it orchestrates - * gets its own {@link StreamsBuilderFactoryBean} and {@link StreamsConfig} - * - * @author Soby Chacko - * @author Lei Chen - * @author Gary Russell - */ -class KafkaStreamsStreamListenerSetupMethodOrchestrator extends AbstractKafkaStreamsBinderProcessor - implements StreamListenerSetupMethodOrchestrator { - - private static final Log LOG = LogFactory - .getLog(KafkaStreamsStreamListenerSetupMethodOrchestrator.class); - - private final StreamListenerParameterAdapter streamListenerParameterAdapter; - - private final Collection streamListenerResultAdapters; - - private final BindingServiceProperties bindingServiceProperties; - - private final KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties; - - private final KeyValueSerdeResolver keyValueSerdeResolver; - - private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; - - private final Map> registeredStoresPerMethod = new HashMap<>(); - - private final Map methodStreamsBuilderFactoryBeanMap = new HashMap<>(); - - StreamsBuilderFactoryBeanConfigurer customizer; - - private final ConfigurableEnvironment environment; - - KafkaStreamsStreamListenerSetupMethodOrchestrator( - BindingServiceProperties bindingServiceProperties, - KafkaStreamsExtendedBindingProperties extendedBindingProperties, - KeyValueSerdeResolver keyValueSerdeResolver, - KafkaStreamsBindingInformationCatalogue bindingInformationCatalogue, - StreamListenerParameterAdapter streamListenerParameterAdapter, - Collection listenerResultAdapters, - CleanupConfig cleanupConfig, - StreamsBuilderFactoryBeanConfigurer customizer, - ConfigurableEnvironment environment) { - super(bindingServiceProperties, bindingInformationCatalogue, extendedBindingProperties, keyValueSerdeResolver, cleanupConfig); - this.bindingServiceProperties = bindingServiceProperties; - this.kafkaStreamsExtendedBindingProperties = extendedBindingProperties; - this.keyValueSerdeResolver = keyValueSerdeResolver; - this.kafkaStreamsBindingInformationCatalogue = bindingInformationCatalogue; - this.streamListenerParameterAdapter = streamListenerParameterAdapter; - this.streamListenerResultAdapters = listenerResultAdapters; - this.customizer = customizer; - this.environment = environment; - } - - @Override - public boolean supports(Method method) { - return methodParameterSupports(method) && (methodReturnTypeSuppports(method) - || Void.TYPE.equals(method.getReturnType())); - } - - private boolean methodReturnTypeSuppports(Method method) { - Class returnType = method.getReturnType(); - if (returnType.equals(KStream.class) || (returnType.isArray() - && returnType.getComponentType().equals(KStream.class))) { - return true; - } - return false; - } - - private boolean methodParameterSupports(Method method) { - boolean supports = false; - for (int i = 0; i < method.getParameterCount(); i++) { - MethodParameter methodParameter = MethodParameter.forExecutable(method, i); - Class parameterType = methodParameter.getParameterType(); - if (parameterType.equals(KStream.class) || parameterType.equals(KTable.class) - || parameterType.equals(GlobalKTable.class)) { - supports = true; - } - } - return supports; - } - - @Override - @SuppressWarnings({"rawtypes", "unchecked"}) - public void orchestrateStreamListenerSetupMethod(StreamListener streamListener, - Method method, Object bean) { - String[] methodAnnotatedOutboundNames = getOutboundBindingTargetNames(method); - validateStreamListenerMethod(streamListener, method, - methodAnnotatedOutboundNames); - String methodAnnotatedInboundName = streamListener.value(); - Object[] adaptedInboundArguments = adaptAndRetrieveInboundArguments(method, - methodAnnotatedInboundName, this.applicationContext, - this.streamListenerParameterAdapter); - try { - ReflectionUtils.makeAccessible(method); - if (Void.TYPE.equals(method.getReturnType())) { - method.invoke(bean, adaptedInboundArguments); - } - else { - Object result = method.invoke(bean, adaptedInboundArguments); - - if (methodAnnotatedOutboundNames != null && methodAnnotatedOutboundNames.length > 0) { - if (result.getClass().isArray()) { - Assert.isTrue( - methodAnnotatedOutboundNames.length == ((Object[]) result).length, - "Result does not match with the number of declared outbounds"); - } - else { - Assert.isTrue(methodAnnotatedOutboundNames.length == 1, - "Result does not match with the number of declared outbounds"); - } - } - - if (methodAnnotatedOutboundNames != null && methodAnnotatedOutboundNames.length > 0) { - methodAnnotatedInboundName = populateInboundIfMissing(method, methodAnnotatedInboundName); - final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsBindingInformationCatalogue - .getStreamsBuilderFactoryBeanPerBinding().get(methodAnnotatedInboundName); - - if (result.getClass().isArray()) { - Object[] outboundKStreams = (Object[]) result; - int i = 0; - for (Object outboundKStream : outboundKStreams) { - final String methodAnnotatedOutboundName = methodAnnotatedOutboundNames[i++]; - - this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding( - methodAnnotatedOutboundName, streamsBuilderFactoryBean); - - Object targetBean = this.applicationContext - .getBean(methodAnnotatedOutboundName); - kafkaStreamsBindingInformationCatalogue.addOutboundKStreamResolvable(targetBean, ResolvableType.forMethodReturnType(method)); - adaptStreamListenerResult(outboundKStream, targetBean); - } - } - else { - this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding( - methodAnnotatedOutboundNames[0], streamsBuilderFactoryBean); - - Object targetBean = this.applicationContext - .getBean(methodAnnotatedOutboundNames[0]); - kafkaStreamsBindingInformationCatalogue.addOutboundKStreamResolvable(targetBean, ResolvableType.forMethodReturnType(method)); - adaptStreamListenerResult(result, targetBean); - } - } - } - } - catch (Exception ex) { - throw new BeanInitializationException( - "Cannot setup StreamListener for " + method, ex); - } - } - - private String populateInboundIfMissing(Method method, String methodAnnotatedInboundName) { - if (!StringUtils.hasText(methodAnnotatedInboundName)) { - Object[] arguments = new Object[method.getParameterTypes().length]; - if (arguments.length > 0) { - MethodParameter methodParameter = MethodParameter.forExecutable(method, 0); - if (methodParameter.hasParameterAnnotation(Input.class)) { - Input methodAnnotation = methodParameter - .getParameterAnnotation(Input.class); - methodAnnotatedInboundName = methodAnnotation.value(); - } - } - } - return methodAnnotatedInboundName; - } - - @SuppressWarnings("unchecked") - private void adaptStreamListenerResult(Object outboundKStream, Object targetBean) { - for (StreamListenerResultAdapter streamListenerResultAdapter : this.streamListenerResultAdapters) { - if (streamListenerResultAdapter.supports( - outboundKStream.getClass(), targetBean.getClass())) { - streamListenerResultAdapter.adapt(outboundKStream, - targetBean); - break; - } - } - } - - @Override - @SuppressWarnings({"unchecked"}) - public Object[] adaptAndRetrieveInboundArguments(Method method, String inboundName, - ApplicationContext applicationContext, - StreamListenerParameterAdapter... adapters) { - Object[] arguments = new Object[method.getParameterTypes().length]; - for (int parameterIndex = 0; parameterIndex < arguments.length; parameterIndex++) { - MethodParameter methodParameter = MethodParameter.forExecutable(method, - parameterIndex); - Class parameterType = methodParameter.getParameterType(); - Object targetReferenceValue = null; - if (methodParameter.hasParameterAnnotation(Input.class)) { - targetReferenceValue = AnnotationUtils - .getValue(methodParameter.getParameterAnnotation(Input.class)); - Input methodAnnotation = methodParameter - .getParameterAnnotation(Input.class); - inboundName = methodAnnotation.value(); - } - else if (arguments.length == 1 && StringUtils.hasText(inboundName)) { - targetReferenceValue = inboundName; - } - if (targetReferenceValue != null) { - Assert.isInstanceOf(String.class, targetReferenceValue, - "Annotation value must be a String"); - Object targetBean = applicationContext - .getBean((String) targetReferenceValue); - BindingProperties bindingProperties = this.bindingServiceProperties - .getBindingProperties(inboundName); - // Retrieve the StreamsConfig created for this method if available. - // Otherwise, create the StreamsBuilderFactory and get the underlying - // config. - if (!this.methodStreamsBuilderFactoryBeanMap.containsKey(method)) { - StreamsBuilderFactoryBean streamsBuilderFactoryBean = buildStreamsBuilderAndRetrieveConfig(method.getDeclaringClass().getSimpleName() + "-" + method.getName(), - applicationContext, - inboundName, null, customizer, this.environment, bindingProperties); - this.methodStreamsBuilderFactoryBeanMap.put(method, streamsBuilderFactoryBean); - } - try { - StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.methodStreamsBuilderFactoryBeanMap - .get(method); - StreamsBuilder streamsBuilder = streamsBuilderFactoryBean.getObject(); - final String applicationId = streamsBuilderFactoryBean.getStreamsConfiguration().getProperty(StreamsConfig.APPLICATION_ID_CONFIG); - KafkaStreamsConsumerProperties extendedConsumerProperties = this.kafkaStreamsExtendedBindingProperties - .getExtendedConsumerProperties(inboundName); - extendedConsumerProperties.setApplicationId(applicationId); - // get state store spec - KafkaStreamsStateStoreProperties spec = buildStateStoreSpec(method); - - Serde keySerde = this.keyValueSerdeResolver - .getInboundKeySerde(extendedConsumerProperties, ResolvableType.forMethodParameter(methodParameter)); - LOG.info("Key Serde used for " + targetReferenceValue + ": " + keySerde.getClass().getName()); - - Serde valueSerde = bindingServiceProperties.getConsumerProperties(inboundName).isUseNativeDecoding() ? - getValueSerde(inboundName, extendedConsumerProperties, ResolvableType.forMethodParameter(methodParameter)) : Serdes.ByteArray(); - LOG.info("Value Serde used for " + targetReferenceValue + ": " + valueSerde.getClass().getName()); - - Topology.AutoOffsetReset autoOffsetReset = getAutoOffsetReset(inboundName, extendedConsumerProperties); - - if (parameterType.isAssignableFrom(KStream.class)) { - KStream stream = getkStream(inboundName, spec, - bindingProperties, extendedConsumerProperties, streamsBuilder, keySerde, valueSerde, - autoOffsetReset, parameterIndex == 0); - KStreamBoundElementFactory.KStreamWrapper kStreamWrapper = (KStreamBoundElementFactory.KStreamWrapper) targetBean; - // wrap the proxy created during the initial target type binding - // with real object (KStream) - kStreamWrapper.wrap((KStream) stream); - this.kafkaStreamsBindingInformationCatalogue.addKeySerde(stream, keySerde); - BindingProperties bindingProperties1 = this.kafkaStreamsBindingInformationCatalogue.getBindingProperties().get(kStreamWrapper); - this.kafkaStreamsBindingInformationCatalogue.registerBindingProperties(stream, bindingProperties1); - - this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding(inboundName, streamsBuilderFactoryBean); - this.kafkaStreamsBindingInformationCatalogue.addConsumerPropertiesPerSbfb(streamsBuilderFactoryBean, - bindingServiceProperties.getConsumerProperties(inboundName)); - - for (StreamListenerParameterAdapter streamListenerParameterAdapter : adapters) { - if (streamListenerParameterAdapter.supports(stream.getClass(), - methodParameter)) { - arguments[parameterIndex] = streamListenerParameterAdapter - .adapt(stream, methodParameter); - break; - } - } - if (arguments[parameterIndex] == null - && parameterType.isAssignableFrom(stream.getClass())) { - arguments[parameterIndex] = stream; - } - Assert.notNull(arguments[parameterIndex], - "Cannot convert argument " + parameterIndex + " of " - + method + "from " + stream.getClass() + " to " - + parameterType); - } - else { - handleKTableGlobalKTableInputs(arguments, parameterIndex, inboundName, parameterType, targetBean, streamsBuilderFactoryBean, - streamsBuilder, extendedConsumerProperties, keySerde, valueSerde, autoOffsetReset, parameterIndex == 0); - } - } - catch (Exception ex) { - throw new IllegalStateException(ex); - } - } - else { - throw new IllegalStateException( - StreamListenerErrorMessages.INVALID_DECLARATIVE_METHOD_PARAMETERS); - } - } - return arguments; - } - - private StoreBuilder buildStateStore(KafkaStreamsStateStoreProperties spec) { - try { - - Serde keySerde = this.keyValueSerdeResolver - .getStateStoreKeySerde(spec.getKeySerdeString()); - Serde valueSerde = this.keyValueSerdeResolver - .getStateStoreValueSerde(spec.getValueSerdeString()); - StoreBuilder builder; - switch (spec.getType()) { - case KEYVALUE: - builder = Stores.keyValueStoreBuilder( - Stores.persistentKeyValueStore(spec.getName()), keySerde, - valueSerde); - break; - case WINDOW: - builder = Stores - .windowStoreBuilder( - Stores.persistentWindowStore(spec.getName(), - Duration.ofMillis(spec.getRetention()), Duration.ofMillis(3), false), - keySerde, valueSerde); - break; - case SESSION: - builder = Stores.sessionStoreBuilder(Stores.persistentSessionStore( - spec.getName(), Duration.ofMillis(spec.getRetention())), keySerde, valueSerde); - break; - default: - throw new UnsupportedOperationException( - "state store type (" + spec.getType() + ") is not supported!"); - } - if (spec.isCacheEnabled()) { - builder = builder.withCachingEnabled(); - } - if (spec.isLoggingDisabled()) { - builder = builder.withLoggingDisabled(); - } - return builder; - } - catch (Exception ex) { - LOG.error("failed to build state store exception : " + ex); - throw ex; - } - } - - private KStream getkStream(String inboundName, - KafkaStreamsStateStoreProperties storeSpec, - BindingProperties bindingProperties, - KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, StreamsBuilder streamsBuilder, - Serde keySerde, Serde valueSerde, - Topology.AutoOffsetReset autoOffsetReset, boolean firstBuild) { - if (storeSpec != null) { - StoreBuilder storeBuilder = buildStateStore(storeSpec); - streamsBuilder.addStateStore(storeBuilder); - if (LOG.isInfoEnabled()) { - LOG.info("state store " + storeBuilder.name() + " added to topology"); - } - } - return getKStream(inboundName, bindingProperties, kafkaStreamsConsumerProperties, streamsBuilder, - keySerde, valueSerde, autoOffsetReset, firstBuild); - } - - private void validateStreamListenerMethod(StreamListener streamListener, - Method method, String[] methodAnnotatedOutboundNames) { - String methodAnnotatedInboundName = streamListener.value(); - if (methodAnnotatedOutboundNames != null) { - for (String s : methodAnnotatedOutboundNames) { - if (StringUtils.hasText(s)) { - Assert.isTrue(isDeclarativeOutput(method, s), - "Method must be declarative"); - } - } - } - if (StringUtils.hasText(methodAnnotatedInboundName)) { - int methodArgumentsLength = method.getParameterTypes().length; - - for (int parameterIndex = 0; parameterIndex < methodArgumentsLength; parameterIndex++) { - MethodParameter methodParameter = MethodParameter.forExecutable(method, - parameterIndex); - Assert.isTrue( - isDeclarativeInput(methodAnnotatedInboundName, methodParameter), - "Method must be declarative"); - } - } - } - - @SuppressWarnings("unchecked") - private boolean isDeclarativeOutput(Method m, String targetBeanName) { - boolean declarative; - Class returnType = m.getReturnType(); - if (returnType.isArray()) { - Class targetBeanClass = this.applicationContext.getType(targetBeanName); - declarative = this.streamListenerResultAdapters.stream() - .anyMatch((slpa) -> slpa.supports(returnType.getComponentType(), - targetBeanClass)); - return declarative; - } - Class targetBeanClass = this.applicationContext.getType(targetBeanName); - declarative = this.streamListenerResultAdapters.stream() - .anyMatch((slpa) -> slpa.supports(returnType, targetBeanClass)); - return declarative; - } - - @SuppressWarnings("unchecked") - private boolean isDeclarativeInput(String targetBeanName, - MethodParameter methodParameter) { - if (!methodParameter.getParameterType().isAssignableFrom(Object.class) - && this.applicationContext.containsBean(targetBeanName)) { - Class targetBeanClass = this.applicationContext.getType(targetBeanName); - if (targetBeanClass != null) { - boolean supports = KafkaStreamsBinderUtils.supportsKStream(methodParameter, targetBeanClass); - if (!supports) { - supports = KTable.class.isAssignableFrom(targetBeanClass) - && KTable.class.isAssignableFrom(methodParameter.getParameterType()); - if (!supports) { - supports = GlobalKTable.class.isAssignableFrom(targetBeanClass) - && GlobalKTable.class.isAssignableFrom(methodParameter.getParameterType()); - } - } - return supports; - } - } - return false; - } - - private static String[] getOutboundBindingTargetNames(Method method) { - SendTo sendTo = AnnotationUtils.findAnnotation(method, SendTo.class); - if (sendTo != null) { - Assert.isTrue(!ObjectUtils.isEmpty(sendTo.value()), - StreamListenerErrorMessages.ATLEAST_ONE_OUTPUT); - Assert.isTrue(sendTo.value().length >= 1, - "At least one outbound destination need to be provided."); - return sendTo.value(); - } - return null; - } - - @SuppressWarnings({"unchecked"}) - private KafkaStreamsStateStoreProperties buildStateStoreSpec(Method method) { - if (!this.registeredStoresPerMethod.containsKey(method)) { - KafkaStreamsStateStore spec = AnnotationUtils.findAnnotation(method, - KafkaStreamsStateStore.class); - if (spec != null) { - Assert.isTrue(!ObjectUtils.isEmpty(spec.name()), "name cannot be empty"); - Assert.isTrue(spec.name().length() >= 1, "name cannot be empty."); - this.registeredStoresPerMethod.put(method, new ArrayList<>()); - this.registeredStoresPerMethod.get(method).add(spec.name()); - KafkaStreamsStateStoreProperties props = new KafkaStreamsStateStoreProperties(); - props.setName(spec.name()); - props.setType(spec.type()); - props.setLength(spec.lengthMs()); - props.setKeySerdeString(spec.keySerde()); - props.setRetention(spec.retentionMs()); - props.setValueSerdeString(spec.valueSerde()); - props.setCacheEnabled(spec.cache()); - props.setLoggingDisabled(!spec.logging()); - return props; - } - } - return null; - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/StreamsBuilderFactoryManager.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/StreamsBuilderFactoryManager.java index 9f2bbf0f6..c0fa13e49 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/StreamsBuilderFactoryManager.java +++ b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/StreamsBuilderFactoryManager.java @@ -39,8 +39,7 @@ import org.springframework.kafka.streams.KafkaStreamsMicrometerListener; * This {@link SmartLifecycle} class ensures that the bean created from it is started very * late through the bootstrap process by setting the phase value closer to * Integer.MAX_VALUE. This is to guarantee that the {@link StreamsBuilderFactoryBean} on a - * {@link org.springframework.cloud.stream.annotation.StreamListener} method with multiple - * bindings is only started after all the binding phases have completed successfully. + * function with multiple bindings is only started after all the binding phases have completed successfully. * * @author Soby Chacko */ diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsProcessor.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsProcessor.java deleted file mode 100644 index 36df1f43f..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsProcessor.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2017-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.annotations; - -import org.apache.kafka.streams.kstream.KStream; - -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; - -/** - * Bindable interface for {@link KStream} input and output. - * - * This interface can be used as a bindable interface with - * {@link org.springframework.cloud.stream.annotation.EnableBinding} when both input and - * output types are single KStream. In other scenarios where multiple types are required, - * other similar bindable interfaces can be created and used. For example, there are cases - * in which multiple KStreams are required on the outbound in the case of KStream - * branching or multiple input types are required either in the form of multiple KStreams - * and a combination of KStreams and KTables. In those cases, new bindable interfaces - * compatible with the requirements must be created. Here are some examples. - * - *

- *     interface KStreamBranchProcessor {
- *         @Input("input")
- *         KStream<?, ?> input();
- *
- *         @Output("output-1")
- *         KStream<?, ?> output1();
- *
- *         @Output("output-2")
- *         KStream<?, ?> output2();
- *
- *         @Output("output-3")
- *         KStream<?, ?> output3();
- *
- *         ......
- *
- *     }
- *
- * - *
- *     interface KStreamKtableProcessor {
- *         @Input("input-1")
- *         KStream<?, ?> input1();
- *
- *         @Input("input-2")
- *         KTable<?, ?> input2();
- *
- *         @Output("output")
- *         KStream<?, ?> output();
- *
- *         ......
- *
- *     }
- *
- * - * @author Marius Bogoevici - * @author Soby Chacko - */ -public interface KafkaStreamsProcessor { - - /** - * Input binding. - * @return {@link Input} binding for {@link KStream} type. - */ - @Input("input") - KStream input(); - - /** - * Output binding. - * @return {@link Output} binding for {@link KStream} type. - */ - @Output("output") - KStream output(); - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsStateStore.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsStateStore.java deleted file mode 100644 index a24cda301..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsStateStore.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.annotations; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsStateStoreProperties; - -/** - * Interface for Kafka Stream state store. - * - * This interface can be used to inject a state store specification into KStream building - * process so that the desired store can be built by StreamBuilder and added to topology - * for later use by processors. This is particularly useful when need to combine stream - * DSL with low level processor APIs. In those cases, if a writable state store is desired - * in processors, it needs to be created using this annotation. Here is the example. - * - *
- *     @StreamListener("input")
- *     @KafkaStreamsStateStore(name="mystate", type= KafkaStreamsStateStoreProperties.StoreType.WINDOW,
- *     								size=300000)
- *	   public void process(KStream<Object, Product> input) {
- *         ......
- *     }
- * 
- * - * With that, you should be able to read/write this state store in your - * processor/transformer code. - * - *
- * 		new Processor<Object, Product>() {
- * 			WindowStore<Object, String> state;
- * 			@Override
- *			public void init(ProcessorContext processorContext) {
- *			state = (WindowStore)processorContext.getStateStore("mystate");
- *				......
- *			}
- *		}
- * 
- * - * @author Lei Chen - */ - -@Target({ ElementType.TYPE, ElementType.METHOD, ElementType.ANNOTATION_TYPE }) -@Retention(RetentionPolicy.RUNTIME) - -public @interface KafkaStreamsStateStore { - - /** - * Provides name of the state store. - * @return name of state store. - */ - String name() default ""; - - /** - * State store type. - * @return {@link KafkaStreamsStateStoreProperties.StoreType} of state store. - */ - KafkaStreamsStateStoreProperties.StoreType type() default KafkaStreamsStateStoreProperties.StoreType.KEYVALUE; - - /** - * Serde used for key. - * @return key serde of state store. - */ - String keySerde() default "org.apache.kafka.common.serialization.Serdes$StringSerde"; - - /** - * Serde used for value. - * @return value serde of state store. - */ - String valueSerde() default "org.apache.kafka.common.serialization.Serdes$StringSerde"; - - /** - * Length in milli-second of Windowed store window. - * @return length in milli-second of window(for windowed store). - */ - long lengthMs() default 0; - - /** - * Retention period for Windowed store windows. - * @return the maximum period of time in milli-second to keep each window in this - * store(for windowed store). - */ - long retentionMs() default 0; - - /** - * Whether catching is enabled or not. - * @return whether caching should be enabled on the created store. - */ - boolean cache() default false; - - /** - * Whether logging is enabled or not. - * @return whether logging should be enabled on the created store. - */ - boolean logging() default true; - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsStateStoreProperties.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsStateStoreProperties.java deleted file mode 100644 index c51cc6120..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsStateStoreProperties.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.properties; - -/** - * Properties for Kafka Streams state store. - * - * @author Lei Chen - */ -public class KafkaStreamsStateStoreProperties { - - /** - * Enumeration for store type. - */ - public enum StoreType { - - /** - * Key value store. - */ - KEYVALUE("keyvalue"), - /** - * Window store. - */ - WINDOW("window"), - /** - * Session store. - */ - SESSION("session"); - - private final String type; - - StoreType(final String type) { - this.type = type; - } - - @Override - public String toString() { - return this.type; - } - - } - - /** - * Name for this state store. - */ - private String name; - - /** - * Type for this state store. - */ - private StoreType type; - - /** - * Size/length of this state store in ms. Only applicable for window store. - */ - private long length; - - /** - * Retention period for this state store in ms. - */ - private long retention; - - /** - * Key serde class specified per state store. - */ - private String keySerdeString; - - /** - * Value serde class specified per state store. - */ - private String valueSerdeString; - - /** - * Whether caching is enabled on this state store. - */ - private boolean cacheEnabled; - - /** - * Whether logging is enabled on this state store. - */ - private boolean loggingDisabled; - - public String getName() { - return this.name; - } - - public void setName(String name) { - this.name = name; - } - - public StoreType getType() { - return this.type; - } - - public void setType(StoreType type) { - this.type = type; - } - - public long getLength() { - return this.length; - } - - public void setLength(long length) { - this.length = length; - } - - public long getRetention() { - return this.retention; - } - - public void setRetention(long retention) { - this.retention = retention; - } - - public String getKeySerdeString() { - return this.keySerdeString; - } - - public void setKeySerdeString(String keySerdeString) { - this.keySerdeString = keySerdeString; - } - - public String getValueSerdeString() { - return this.valueSerdeString; - } - - public void setValueSerdeString(String valueSerdeString) { - this.valueSerdeString = valueSerdeString; - } - - public boolean isCacheEnabled() { - return this.cacheEnabled; - } - - public void setCacheEnabled(boolean cacheEnabled) { - this.cacheEnabled = cacheEnabled; - } - - public boolean isLoggingDisabled() { - return this.loggingDisabled; - } - - public void setLoggingDisabled(boolean loggingDisabled) { - this.loggingDisabled = loggingDisabled; - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CompositeNonNativeSerde.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CompositeNonNativeSerde.java deleted file mode 100644 index a7b0300ac..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CompositeNonNativeSerde.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.serde; - -import org.springframework.messaging.converter.CompositeMessageConverter; - -/** - * This class provides the same functionality as {@link MessageConverterDelegateSerde} and is deprecated. - * It is kept for backward compatibility reasons and will be removed in version 3.1 - * - * @author Soby Chacko - * @since 2.1 - * - * @deprecated in favor of {@link MessageConverterDelegateSerde} - */ -@Deprecated -public class CompositeNonNativeSerde extends MessageConverterDelegateSerde { - - public CompositeNonNativeSerde(CompositeMessageConverter compositeMessageConverter) { - super(compositeMessageConverter); - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/MessageConverterDelegateSerde.java b/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/MessageConverterDelegateSerde.java deleted file mode 100644 index a64e28695..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/MessageConverterDelegateSerde.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright 2019-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.serde; - -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; - -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serializer; - -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.converter.CompositeMessageConverter; -import org.springframework.messaging.converter.MessageConverter; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.util.Assert; -import org.springframework.util.MimeType; -import org.springframework.util.MimeTypeUtils; - -/** - * A {@link Serde} implementation that wraps the list of {@link MessageConverter}s from - * {@link CompositeMessageConverter}. - * - * The primary motivation for this class is to provide an avro based {@link Serde} that is - * compatible with the schema registry that Spring Cloud Stream provides. When using the - * schema registry support from Spring Cloud Stream in a Kafka Streams binder based - * application, the applications can deserialize the incoming Kafka Streams records using - * the built in Avro {@link MessageConverter}. However, this same message conversion - * approach will not work downstream in other operations in the topology for Kafka Streams - * as some of them needs a {@link Serde} instance that can talk to the Spring Cloud Stream - * provided Schema Registry. This implementation will solve that problem. - * - * Only Avro and JSON based converters are exposed as binder provided {@link Serde} - * implementations currently. - * - * Users of this class must call the - * {@link MessageConverterDelegateSerde#configure(Map, boolean)} method to configure the - * {@link Serde} object. At the very least the configuration map must include a key called - * "valueClass" to indicate the type of the target object for deserialization. If any - * other content type other than JSON is needed (only Avro is available now other than - * JSON), that needs to be included in the configuration map with the key "contentType". - * For example, - * - *
- * Map<String, Object> config = new HashMap<>();
- * config.put("valueClass", Foo.class);
- * config.put("contentType", "application/avro");
- * 
- * - * Then use the above map when calling the configure method. - * - * This class is only intended to be used when writing a Spring Cloud Stream Kafka Streams - * application that uses Spring Cloud Stream schema registry for schema evolution. - * - * An instance of this class is provided as a bean by the binder configuration and - * typically the applications can autowire that bean. This is the expected usage pattern - * of this class. - * - * @param type of the object to marshall - * @author Soby Chacko - * @since 3.0 - * @deprecated in favor of other schema registry providers instead of Spring Cloud Schema Registry. See its motivation above. - */ -@Deprecated -public class MessageConverterDelegateSerde implements Serde { - - private static final String VALUE_CLASS_HEADER = "valueClass"; - - private static final String AVRO_FORMAT = "avro"; - - private static final MimeType DEFAULT_AVRO_MIME_TYPE = new MimeType("application", - "*+" + AVRO_FORMAT); - - private final MessageConverterDelegateDeserializer messageConverterDelegateDeserializer; - - private final MessageConverterDelegateSerializer messageConverterDelegateSerializer; - - public MessageConverterDelegateSerde( - CompositeMessageConverter compositeMessageConverter) { - this.messageConverterDelegateDeserializer = new MessageConverterDelegateDeserializer<>( - compositeMessageConverter); - this.messageConverterDelegateSerializer = new MessageConverterDelegateSerializer<>( - compositeMessageConverter); - } - - @Override - public void configure(Map configs, boolean isKey) { - this.messageConverterDelegateDeserializer.configure(configs, isKey); - this.messageConverterDelegateSerializer.configure(configs, isKey); - } - - @Override - public void close() { - // No-op - } - - @Override - public Serializer serializer() { - return this.messageConverterDelegateSerializer; - } - - @Override - public Deserializer deserializer() { - return this.messageConverterDelegateDeserializer; - } - - private static MimeType resolveMimeType(Map configs) { - if (configs.containsKey(MessageHeaders.CONTENT_TYPE)) { - String contentType = (String) configs.get(MessageHeaders.CONTENT_TYPE); - if (DEFAULT_AVRO_MIME_TYPE.equals(MimeTypeUtils.parseMimeType(contentType))) { - return DEFAULT_AVRO_MIME_TYPE; - } - else if (contentType.contains("avro")) { - return MimeTypeUtils.parseMimeType("application/avro"); - } - else { - return new MimeType("application", "json", StandardCharsets.UTF_8); - } - } - else { - return new MimeType("application", "json", StandardCharsets.UTF_8); - } - } - - /** - * Custom {@link Deserializer} that uses the {@link org.springframework.cloud.stream.converter.CompositeMessageConverterFactory}. - * - * @param parameterized target type for deserialization - */ - private static class MessageConverterDelegateDeserializer implements Deserializer { - - private final MessageConverter messageConverter; - - private MimeType mimeType; - - private Class valueClass; - - MessageConverterDelegateDeserializer( - CompositeMessageConverter compositeMessageConverter) { - this.messageConverter = compositeMessageConverter; - } - - @Override - public void configure(Map configs, boolean isKey) { - Assert.isTrue(configs.containsKey(VALUE_CLASS_HEADER), - "Deserializers must provide a configuration for valueClass."); - final Object valueClass = configs.get(VALUE_CLASS_HEADER); - Assert.isTrue(valueClass instanceof Class, - "Deserializers must provide a valid value for valueClass."); - this.valueClass = (Class) valueClass; - this.mimeType = resolveMimeType(configs); - } - - @SuppressWarnings("unchecked") - @Override - public U deserialize(String topic, byte[] data) { - Message message = MessageBuilder.withPayload(data) - .setHeader(MessageHeaders.CONTENT_TYPE, this.mimeType.toString()) - .build(); - U messageConverted = (U) this.messageConverter.fromMessage(message, - this.valueClass); - Assert.notNull(messageConverted, "Deserialization failed."); - return messageConverted; - } - - @Override - public void close() { - // No-op - } - - } - - /** - * Custom {@link Serializer} that uses the {@link org.springframework.cloud.stream.converter.CompositeMessageConverterFactory}. - * - * @param parameterized type for serialization - */ - private static class MessageConverterDelegateSerializer implements Serializer { - - private final MessageConverter messageConverter; - - private MimeType mimeType; - - MessageConverterDelegateSerializer( - CompositeMessageConverter compositeMessageConverter) { - this.messageConverter = compositeMessageConverter; - } - - @Override - public void configure(Map configs, boolean isKey) { - this.mimeType = resolveMimeType(configs); - } - - @Override - public byte[] serialize(String topic, V data) { - Message message = MessageBuilder.withPayload(data).build(); - Map headers = new HashMap<>(message.getHeaders()); - headers.put(MessageHeaders.CONTENT_TYPE, this.mimeType.toString()); - MessageHeaders messageHeaders = new MessageHeaders(headers); - final Object payload = this.messageConverter - .toMessage(message.getPayload(), messageHeaders).getPayload(); - return (byte[]) payload; - } - - @Override - public void close() { - // No-op - } - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionCompositionTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionCompositionTests.java index f3716d414..cebe3121a 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionCompositionTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionCompositionTests.java @@ -224,6 +224,7 @@ public class KafkaStreamsFunctionCompositionTests { try (ConfigurableApplicationContext context = app.run( "--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.kafka.streams.binder.applicationId=my-app-id", "--spring.cloud.stream.function.definition=fooBiFunc|anotherFooFunc|yetAnotherFooFunc|lastFunctionInChain", "--spring.cloud.stream.function.bindings.fooBiFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-in-0=input1", "--spring.cloud.stream.function.bindings.fooBiFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-in-1=input2", @@ -266,6 +267,7 @@ public class KafkaStreamsFunctionCompositionTests { try (ConfigurableApplicationContext context = app.run( "--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.kafka.streams.binder.applicationId=my-app-id-xyz", "--spring.cloud.stream.function.definition=curriedFunc|anotherFooFunc|yetAnotherFooFunc|lastFunctionInChain", "--spring.cloud.stream.function.bindings.curriedFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-in-0=input1", "--spring.cloud.stream.function.bindings.curriedFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-in-1=input2", diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java index 48c7f5f28..4eab47fbe 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java @@ -19,6 +19,7 @@ package org.springframework.cloud.stream.binder.kafka.streams; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.function.Function; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -41,7 +42,6 @@ import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Test; import org.mockito.Mockito; @@ -49,9 +49,6 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.kafka.KafkaProperties; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; @@ -64,9 +61,9 @@ import org.springframework.kafka.support.serializer.JsonSerde; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.internal.verification.VerificationModeFactory.times; /** @@ -150,22 +147,23 @@ public class KafkaStreamsInteractiveQueryIntegrationTests { QueryableStoreType> storeType = QueryableStoreTypes.keyValueStore(); final StringSerializer serializer = new StringSerializer(); try { - interactiveQueryService.getHostInfo("foo", "fooKey", serializer); + interactiveQueryService.getHostInfo("foo", "foobarApp-key", serializer); } catch (Exception ignored) { } Mockito.verify(mockKafkaStreams, times(3)) - .queryMetadataForKey("foo", "fooKey", serializer); + .queryMetadataForKey("foo", "foobarApp-key", serializer); } @Test - @Ignore - public void testKstreamBinderWithPojoInputAndStringOuput() throws Exception { + public void testKstreamBinderWithPojoInputAndStringOuput() { SpringApplication app = new SpringApplication(ProductCountApplication.class); app.setWebApplicationType(WebApplicationType.NONE); ConfigurableApplicationContext context = app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input", + "--spring.cloud.stream.function.bindings.process-out-0=output", "--spring.cloud.stream.bindings.input.destination=foos", "--spring.cloud.stream.bindings.output.destination=counts-id", "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", @@ -226,27 +224,25 @@ public class KafkaStreamsInteractiveQueryIntegrationTests { assertThat(hostInfo.host() + ":" + hostInfo.port()) .isEqualTo(embeddedKafka.getBrokersAsString()); - HostInfo hostInfoFoo = interactiveQueryService - .getHostInfo("prod-id-count-store-foo", 123, new IntegerSerializer()); - assertThat(hostInfoFoo).isNull(); + assertThatThrownBy(() -> interactiveQueryService + .getHostInfo("prod-id-count-store-foo", 123, new IntegerSerializer())) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Error when retrieving state store."); final List hostInfos = interactiveQueryService.getAllHostsInfo("prod-id-count-store"); assertThat(hostInfos.size()).isEqualTo(1); final HostInfo hostInfo1 = hostInfos.get(0); assertThat(hostInfo1.host() + ":" + hostInfo1.port()) .isEqualTo(embeddedKafka.getBrokersAsString()); - } - @EnableBinding(KafkaStreamsProcessor.class) @EnableAutoConfiguration public static class ProductCountApplication { - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { + @Bean + public Function, KStream> process() { - return input.filter((key, product) -> product.getId() == 123) + return input -> input.filter((key, product) -> product.getId() == 123) .map((key, value) -> new KeyValue<>(value.id, value)) .groupByKey(Grouped.with(new Serdes.IntegerSerde(), new JsonSerde<>(Product.class))) diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializationErrorHandlerByKafkaTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializationErrorHandlerByKafkaTests.java deleted file mode 100644 index a2323c555..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializationErrorHandlerByKafkaTests.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.integration; - -import java.time.Duration; -import java.util.Arrays; -import java.util.Map; - -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.kstream.Grouped; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.Materialized; -import org.apache.kafka.streams.kstream.TimeWindows; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; - -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; -import org.springframework.cloud.stream.binder.kafka.utils.DlqPartitionFunction; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.PropertySource; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.test.EmbeddedKafkaBroker; -import org.springframework.kafka.test.rule.EmbeddedKafkaRule; -import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; - -/** - * @author Soby Chacko - */ -@RunWith(SpringRunner.class) -@ContextConfiguration -@DirtiesContext -public abstract class DeserializationErrorHandlerByKafkaTests { - - @ClassRule - public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, - "abc-DeserializationErrorHandlerByKafkaTests-In", - "xyz-DeserializationErrorHandlerByKafkaTests-In", - "DeserializationErrorHandlerByKafkaTests-out", - "error.abc-DeserializationErrorHandlerByKafkaTests-In.group", - "error.xyz-DeserializationErrorHandlerByKafkaTests-In.group", - "error.word1.groupx", - "error.word2.groupx"); - - private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule - .getEmbeddedKafka(); - - @SpyBean - org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsMessageConversionDelegate conversionDelegate; - - private static Consumer consumer; - - @BeforeClass - public static void setUp() { - System.setProperty("spring.cloud.stream.kafka.streams.binder.brokers", - embeddedKafka.getBrokersAsString()); - - System.setProperty("server.port", "0"); - System.setProperty("spring.jmx.enabled", "false"); - - Map consumerProps = KafkaTestUtils.consumerProps("fooc", "false", - embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - consumer = cf.createConsumer(); - embeddedKafka.consumeFromEmbeddedTopics(consumer, "DeserializationErrorHandlerByKafkaTests-out", "DeserializationErrorHandlerByKafkaTests-out"); - } - - @AfterClass - public static void tearDown() { - consumer.close(); - System.clearProperty("spring.cloud.stream.kafka.streams.binder.brokers"); - System.clearProperty("server.port"); - System.clearProperty("spring.jmx.enabled"); - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.destination=abc-DeserializationErrorHandlerByKafkaTests-In", - "spring.cloud.stream.bindings.output.destination=DeserializationErrorHandlerByKafkaTests-Out", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=deser-kafka-dlq", - "spring.cloud.stream.bindings.input.group=group", - "spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler=sendToDlq", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.valueSerde=" - + "org.apache.kafka.common.serialization.Serdes$IntegerSerde" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - public static class DeserializationByKafkaAndDlqTests - extends DeserializationErrorHandlerByKafkaTests { - - @Test - @Ignore - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("abc-DeserializationErrorHandlerByKafkaTests-In"); - template.sendDefault(1, null, "foobar"); - - Map consumerProps = KafkaTestUtils.consumerProps("foobar", - "false", embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - Consumer consumer1 = cf.createConsumer(); - embeddedKafka.consumeFromAnEmbeddedTopic(consumer1, "error.abc-DeserializationErrorHandlerByKafkaTests-In.group"); - - ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer1, - "error.abc-DeserializationErrorHandlerByKafkaTests-In.group"); - assertThat(cr.value()).isEqualTo("foobar"); - assertThat(cr.partition()).isEqualTo(0); // custom partition function - - // Ensuring that the deserialization was indeed done by Kafka natively - verify(conversionDelegate, never()).deserializeOnInbound(any(Class.class), - any(KStream.class)); - verify(conversionDelegate, never()).serializeOnOutbound(any(KStream.class)); - } - - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.destination=xyz-DeserializationErrorHandlerByKafkaTests-In", - "spring.cloud.stream.bindings.output.destination=DeserializationErrorHandlerByKafkaTests-Out", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=deser-kafka-dlq", - "spring.cloud.stream.bindings.input.group=group", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.deserializationExceptionHandler=sendToDlq", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.valueSerde=" - + "org.apache.kafka.common.serialization.Serdes$IntegerSerde" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - public static class DeserializationByKafkaAndDlqPerBindingTests - extends DeserializationErrorHandlerByKafkaTests { - - @Test - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("xyz-DeserializationErrorHandlerByKafkaTests-In"); - template.sendDefault(1, null, "foobar"); - - Map consumerProps = KafkaTestUtils.consumerProps("foobar", - "false", embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - Consumer consumer1 = cf.createConsumer(); - embeddedKafka.consumeFromAnEmbeddedTopic(consumer1, "error.xyz-DeserializationErrorHandlerByKafkaTests-In.group"); - - ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer1, - "error.xyz-DeserializationErrorHandlerByKafkaTests-In.group"); - assertThat(cr.value()).isEqualTo("foobar"); - assertThat(cr.partition()).isEqualTo(0); // custom partition function - - // Ensuring that the deserialization was indeed done by Kafka natively - verify(conversionDelegate, never()).deserializeOnInbound(any(Class.class), - any(KStream.class)); - verify(conversionDelegate, never()).serializeOnOutbound(any(KStream.class)); - } - - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.destination=word1,word2", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=deser-kafka-dlq-multi-input", - "spring.cloud.stream.bindings.input.group=groupx", - "spring.cloud.stream.kafka.streams.binder.serdeError=sendToDlq", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.valueSerde=" - + "org.apache.kafka.common.serialization.Serdes$IntegerSerde" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - // @checkstyle:on - public static class DeserializationByKafkaAndDlqTestsWithMultipleInputs - extends DeserializationErrorHandlerByKafkaTests { - - @Test - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("word1"); - template.sendDefault("foobar"); - - template.setDefaultTopic("word2"); - template.sendDefault("foobar"); - - Map consumerProps = KafkaTestUtils.consumerProps("foobarx", - "false", embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - Consumer consumer1 = cf.createConsumer(); - embeddedKafka.consumeFromEmbeddedTopics(consumer1, "error.word1.groupx", - "error.word2.groupx"); - - ConsumerRecord cr1 = KafkaTestUtils.getSingleRecord(consumer1, - "error.word1.groupx"); - assertThat(cr1.value()).isEqualTo("foobar"); - ConsumerRecord cr2 = KafkaTestUtils.getSingleRecord(consumer1, - "error.word2.groupx"); - assertThat(cr2.value()).isEqualTo("foobar"); - - // Ensuring that the deserialization was indeed done by Kafka natively - verify(conversionDelegate, never()).deserializeOnInbound(any(Class.class), - any(KStream.class)); - verify(conversionDelegate, never()).serializeOnOutbound(any(KStream.class)); - } - - } - - @EnableBinding(KafkaStreamsProcessor.class) - @EnableAutoConfiguration - @PropertySource("classpath:/org/springframework/cloud/stream/binder/kstream/integTest-1.properties") - public static class WordCountProcessorApplication { - - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - - return input - .flatMapValues( - value -> Arrays.asList(value.toLowerCase().split("\\W+"))) - .map((key, value) -> new KeyValue<>(value, value)) - .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) - .windowedBy(TimeWindows.of(Duration.ofMillis(5000))).count(Materialized.as("foo-WordCounts-x")) - .toStream().map((key, value) -> new KeyValue<>(null, - "Count for " + key.key() + " : " + value)); - } - - @Bean - public DlqPartitionFunction partitionFunction() { - return (group, rec, ex) -> 0; - } - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializtionErrorHandlerByBinderTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializtionErrorHandlerByBinderTests.java deleted file mode 100644 index bf2d9cfce..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializtionErrorHandlerByBinderTests.java +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.integration; - -import java.time.Duration; -import java.util.Map; - -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.kstream.Grouped; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.Materialized; -import org.apache.kafka.streams.kstream.TimeWindows; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; - -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.support.serializer.JsonSerde; -import org.springframework.kafka.test.EmbeddedKafkaBroker; -import org.springframework.kafka.test.rule.EmbeddedKafkaRule; -import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; - -/** - * @author Soby Chacko - */ -@RunWith(SpringRunner.class) -@ContextConfiguration -@DirtiesContext -public abstract class DeserializtionErrorHandlerByBinderTests { - - @ClassRule - public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, - "foos", "goos", - "counts-id", "error.foos.foobar-group", "error.goos.foobar-group", "error.foos1.fooz-group", - "error.foos2.fooz-group"); - - private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule - .getEmbeddedKafka(); - - @SpyBean - org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsMessageConversionDelegate conversionDelegate; - - private static Consumer consumer; - - @BeforeClass - public static void setUp() throws Exception { - System.setProperty("spring.cloud.stream.kafka.streams.binder.brokers", - embeddedKafka.getBrokersAsString()); - System.setProperty("server.port", "0"); - System.setProperty("spring.jmx.enabled", "false"); - - Map consumerProps = KafkaTestUtils.consumerProps("kafka-streams-dlq-tests", "false", - embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - consumer = cf.createConsumer(); - embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "counts-id"); - } - - @AfterClass - public static void tearDown() { - consumer.close(); - System.clearProperty("spring.cloud.stream.kafka.streams.binder.brokers"); - System.clearProperty("server.port"); - System.clearProperty("spring.jmx.enabled"); - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", - "spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", - "spring.cloud.stream.bindings.input.destination=foos", - "spring.cloud.stream.bindings.output.destination=counts-id", - "spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", - "spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" - + "=org.apache.kafka.common.serialization.Serdes$IntegerSerde", - "spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" - + "=org.apache.kafka.common.serialization.Serdes$StringSerde", - "spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler=sendToDlq", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id" - + "=deserializationByBinderAndDlqTests", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.dlqPartitions=1", - "spring.cloud.stream.bindings.input.group=foobar-group" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - public static class DeserializationByBinderAndDlqTests - extends DeserializtionErrorHandlerByBinderTests { - - @Test - @Ignore - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("foos"); - template.sendDefault(1, 7, "hello"); - - Map consumerProps = KafkaTestUtils.consumerProps("foobar", - "false", embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - Consumer consumer1 = cf.createConsumer(); - embeddedKafka.consumeFromAnEmbeddedTopic(consumer1, - "error.foos.foobar-group"); - - ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer1, - "error.foos.foobar-group"); - assertThat(cr.value()).isEqualTo("hello"); - assertThat(cr.partition()).isEqualTo(0); - - // Ensuring that the deserialization was indeed done by the binder - verify(conversionDelegate).deserializeOnInbound(any(Class.class), - any(KStream.class)); - } - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", - "spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", - "spring.cloud.stream.bindings.input.destination=goos", - "spring.cloud.stream.bindings.output.destination=counts-id", - "spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", - "spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" - + "=org.apache.kafka.common.serialization.Serdes$IntegerSerde", - "spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" - + "=org.apache.kafka.common.serialization.Serdes$StringSerde", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.deserializationExceptionHandler=sendToDlq", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id" - + "=deserializationByBinderAndDlqTests", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.dlqPartitions=1", - "spring.cloud.stream.bindings.input.group=foobar-group" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - public static class DeserializationByBinderAndDlqSetOnConsumerBindingTests - extends DeserializtionErrorHandlerByBinderTests { - - @Test - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("goos"); - template.sendDefault(1, 7, "hello"); - - Map consumerProps = KafkaTestUtils.consumerProps("foobar", - "false", embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - Consumer consumer1 = cf.createConsumer(); - embeddedKafka.consumeFromAnEmbeddedTopic(consumer1, - "error.goos.foobar-group"); - - ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer1, - "error.goos.foobar-group"); - assertThat(cr.value()).isEqualTo("hello"); - assertThat(cr.partition()).isEqualTo(0); - - // Ensuring that the deserialization was indeed done by the binder - verify(conversionDelegate).deserializeOnInbound(any(Class.class), - any(KStream.class)); - } - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", - "spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", - "spring.cloud.stream.bindings.input.destination=foos1,foos2", - "spring.cloud.stream.bindings.output.destination=counts-id", - "spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", - "spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" - + "=org.apache.kafka.common.serialization.Serdes$StringSerde", - "spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" - + "=org.apache.kafka.common.serialization.Serdes$StringSerde", - "spring.cloud.stream.kafka.streams.binder.serdeError=sendToDlq", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id" - + "=deserializationByBinderAndDlqTestsWithMultipleInputs", - "spring.cloud.stream.bindings.input.group=fooz-group" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - public static class DeserializationByBinderAndDlqTestsWithMultipleInputs - extends DeserializtionErrorHandlerByBinderTests { - - @Test - @SuppressWarnings("unchecked") - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("foos1"); - template.sendDefault("hello"); - - template.setDefaultTopic("foos2"); - template.sendDefault("hello"); - - Map consumerProps = KafkaTestUtils.consumerProps("foobar1", - "false", embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - Consumer consumer1 = cf.createConsumer(); - embeddedKafka.consumeFromEmbeddedTopics(consumer1, "error.foos1.fooz-group", - "error.foos2.fooz-group"); - - ConsumerRecord cr1 = KafkaTestUtils.getSingleRecord(consumer1, - "error.foos1.fooz-group"); - assertThat(cr1.value().equals("hello")).isTrue(); - - ConsumerRecord cr2 = KafkaTestUtils.getSingleRecord(consumer1, - "error.foos2.fooz-group"); - assertThat(cr2.value().equals("hello")).isTrue(); - - // Ensuring that the deserialization was indeed done by the binder - verify(conversionDelegate).deserializeOnInbound(any(Class.class), - any(KStream.class)); - } - - } - - @EnableBinding(KafkaStreamsProcessor.class) - @EnableAutoConfiguration - public static class ProductCountApplication { - - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - return input.filter((key, product) -> product.getId() == 123) - .map((key, value) -> new KeyValue<>(value, value)) - .groupByKey(Grouped.with(new JsonSerde<>(Product.class), - new JsonSerde<>(Product.class))) - .windowedBy(TimeWindows.of(Duration.ofMillis(5000))) - .count(Materialized.as("id-count-store-x")).toStream() - .map((key, value) -> new KeyValue<>(key.key().id, value)); - } - - } - - static class Product { - - Integer id; - - public Integer getId() { - return id; - } - - public void setId(Integer id) { - this.id = id; - } - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderHealthIndicatorTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderHealthIndicatorTests.java index a0694b152..e1a2f1f3f 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderHealthIndicatorTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderHealthIndicatorTests.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -39,12 +40,7 @@ import org.springframework.boot.actuate.health.CompositeHealthContributor; import org.springframework.boot.actuate.health.Health; import org.springframework.boot.actuate.health.Status; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.cloud.stream.annotation.StreamListener; import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsBinderHealthIndicator; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.kafka.config.KafkaStreamsCustomizer; @@ -56,7 +52,6 @@ import org.springframework.kafka.support.SendResult; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; import org.springframework.util.concurrent.ListenableFuture; import org.springframework.util.concurrent.ListenableFutureCallback; @@ -207,6 +202,8 @@ public class KafkaStreamsBinderHealthIndicatorTests { SpringApplication app = new SpringApplication(KStreamApplication.class); app.setWebApplicationType(WebApplicationType.NONE); return app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input", + "--spring.cloud.stream.function.bindings.process-out-0=output", "--spring.cloud.stream.bindings.input.destination=in", "--spring.cloud.stream.bindings.output.destination=out", "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", @@ -225,6 +222,11 @@ public class KafkaStreamsBinderHealthIndicatorTests { SpringApplication app = new SpringApplication(AnotherKStreamApplication.class); app.setWebApplicationType(WebApplicationType.NONE); return app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.function.definition=process;process2", + "--spring.cloud.stream.function.bindings.process-in-0=input", + "--spring.cloud.stream.function.bindings.process-out-0=output", + "--spring.cloud.stream.function.bindings.process2-in-0=input2", + "--spring.cloud.stream.function.bindings.process2-out-0=output2", "--spring.cloud.stream.bindings.input.destination=in", "--spring.cloud.stream.bindings.output.destination=out", "--spring.cloud.stream.bindings.input2.destination=in2", @@ -242,14 +244,12 @@ public class KafkaStreamsBinderHealthIndicatorTests { + embeddedKafka.getBrokersAsString()); } - @EnableBinding(KafkaStreamsProcessor.class) @EnableAutoConfiguration public static class KStreamApplication { - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - return input.filter((key, product) -> { + @Bean + public Function, KStream> process() { + return input -> input.filter((key, product) -> { if (product.getId() != 123) { throw new IllegalArgumentException(); } @@ -259,14 +259,12 @@ public class KafkaStreamsBinderHealthIndicatorTests { } - @EnableBinding({ KafkaStreamsProcessor.class, KafkaStreamsProcessorX.class }) @EnableAutoConfiguration public static class AnotherKStreamApplication { - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - return input.filter((key, product) -> { + @Bean + public Function, KStream> process() { + return input -> input.filter((key, product) -> { if (product.getId() != 123) { throw new IllegalArgumentException(); } @@ -274,10 +272,9 @@ public class KafkaStreamsBinderHealthIndicatorTests { }); } - @StreamListener("input2") - @SendTo("output2") - public KStream process2(KStream input) { - return input.filter((key, product) -> { + @Bean + public Function, KStream> process2() { + return input -> input.filter((key, product) -> { if (product.getId() != 123) { throw new IllegalArgumentException(); } @@ -300,16 +297,6 @@ public class KafkaStreamsBinderHealthIndicatorTests { } - public interface KafkaStreamsProcessorX { - - @Input("input2") - KStream input(); - - @Output("output2") - KStream output(); - - } - public static class Product { Integer id; @@ -323,5 +310,4 @@ public class KafkaStreamsBinderHealthIndicatorTests { } } - } diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderMultipleInputTopicsTest.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderMultipleInputTopicsTest.java index f857c3494..2b841e77f 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderMultipleInputTopicsTest.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderMultipleInputTopicsTest.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.function.Function; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -37,10 +38,6 @@ import org.junit.Test; import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.kafka.core.CleanupConfig; @@ -50,7 +47,6 @@ import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; import static org.assertj.core.api.Assertions.assertThat; @@ -100,6 +96,8 @@ public class KafkaStreamsBinderMultipleInputTopicsTest { ConfigurableApplicationContext context = app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input", + "--spring.cloud.stream.function.bindings.process-out-0=output", "--spring.cloud.stream.bindings.input.destination=words1,words2", "--spring.cloud.stream.bindings.output.destination=counts", "--spring.cloud.stream.bindings.output.contentType=application/json", @@ -146,21 +144,13 @@ public class KafkaStreamsBinderMultipleInputTopicsTest { assertThat(wordCounts.contains("{\"word\":\"foobar2\",\"count\":1}")).isTrue(); } - @EnableBinding(KafkaStreamsProcessor.class) @EnableAutoConfiguration static class WordCountProcessorApplication { - @StreamListener - @SendTo("output") - public KStream process( - @Input("input") KStream input) { + @Bean + public Function, KStream> process() { - input.map((k, v) -> { - System.out.println(k); - System.out.println(v); - return new KeyValue<>(k, v); - }); - return input + return input -> input .flatMapValues( value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .map((key, value) -> new KeyValue<>(value, value)) diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java index bda892780..d95658cee 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java @@ -18,6 +18,7 @@ package org.springframework.cloud.stream.binder.kafka.streams.integration; import java.time.Duration; import java.util.Map; +import java.util.function.Function; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -36,10 +37,8 @@ import org.junit.Test; import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; import org.springframework.kafka.core.DefaultKafkaProducerFactory; import org.springframework.kafka.core.KafkaTemplate; @@ -47,7 +46,6 @@ import org.springframework.kafka.support.serializer.JsonSerde; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; import static org.assertj.core.api.Assertions.assertThat; @@ -89,6 +87,8 @@ public class KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests { app.setWebApplicationType(WebApplicationType.NONE); ConfigurableApplicationContext context = app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input", + "--spring.cloud.stream.function.bindings.process-out-0=output", "--spring.cloud.stream.bindings.input.destination=foos", "--spring.cloud.stream.bindings.output.destination=counts-id", "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", @@ -122,24 +122,19 @@ public class KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests { assertThat(cr.value()).isEqualTo(1L); } - @EnableBinding(KafkaStreamsProcessor.class) @EnableAutoConfiguration public static class ProductCountApplication { - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - return input.filter((key, product) -> product.getId() == 123) + @Bean + public Function, KStream> process() { + return input -> input.filter((key, product) -> product.getId() == 123) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Grouped.with(new JsonSerde<>(Product.class), new JsonSerde<>(Product.class))) .windowedBy(TimeWindows.of(Duration.ofMillis(5000))) .count(Materialized.as("id-count-store-x")).toStream() - .map((key, value) -> { - return new KeyValue<>(key.key().id, value); - }); + .map((key, value) -> new KeyValue<>(key.key().id, value)); } - } public static class Product { diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsNativeEncodingDecodingTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsNativeEncodingDecodingTests.java deleted file mode 100644 index 4bb5abb46..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsNativeEncodingDecodingTests.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.integration; - -import java.time.Duration; -import java.util.Arrays; -import java.util.Map; - -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.kstream.Grouped; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.Materialized; -import org.apache.kafka.streams.kstream.TimeWindows; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.runner.RunWith; - -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.test.EmbeddedKafkaBroker; -import org.springframework.kafka.test.rule.EmbeddedKafkaRule; -import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.util.StopWatch; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; - -/** - * @author Soby Chacko - */ -@RunWith(SpringRunner.class) -@ContextConfiguration -@DirtiesContext -public abstract class KafkaStreamsNativeEncodingDecodingTests { - - @ClassRule - public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, - "decode-counts", "decode-counts-1"); - - private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule - .getEmbeddedKafka(); - - @SpyBean - org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsMessageConversionDelegate conversionDelegate; - - private static Consumer consumer; - - @BeforeClass - public static void setUp() { - System.setProperty("spring.cloud.stream.kafka.streams.binder.brokers", - embeddedKafka.getBrokersAsString()); - System.setProperty("server.port", "0"); - System.setProperty("spring.jmx.enabled", "false"); - - Map consumerProps = KafkaTestUtils.consumerProps("group", "false", - embeddedKafka); - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - consumer = cf.createConsumer(); - embeddedKafka.consumeFromEmbeddedTopics(consumer, "decode-counts", "decode-counts-1"); - } - - @AfterClass - public static void tearDown() { - consumer.close(); - System.clearProperty("spring.cloud.stream.kafka.streams.binder.brokers"); - System.clearProperty("server.port"); - System.clearProperty("spring.jmx.enabled"); - } - - @SpringBootTest(properties = { - "spring.cloud.stream.bindings.input.destination=decode-words-1", - "spring.cloud.stream.bindings.output.destination=decode-counts-1", - "spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId" - + "=NativeEncodingDecodingEnabledTests-abc" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) - public static class NativeEncodingDecodingEnabledTests - extends KafkaStreamsNativeEncodingDecodingTests { - - @Test - public void test() throws Exception { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("decode-words-1"); - template.sendDefault("foobar"); - ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, - "decode-counts-1"); - assertThat(cr.value().equals("Count for foobar : 1")).isTrue(); - - verify(conversionDelegate, never()).serializeOnOutbound(any(KStream.class)); - verify(conversionDelegate, never()).deserializeOnInbound(any(Class.class), - any(KStream.class)); - } - - } - - @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { - "spring.cloud.stream.bindings.input.destination=decode-words", - "spring.cloud.stream.bindings.output.destination=decode-counts", - "spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", - "spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", - "spring.cloud.stream.kafka.streams.bindings.input3.consumer.applicationId" - + "=hello-NativeEncodingDecodingEnabledTests-xyz" }) - public static class NativeEncodingDecodingDisabledTests - extends KafkaStreamsNativeEncodingDecodingTests { - - @Test - public void test() { - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic("decode-words"); - template.sendDefault("foobar"); - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - System.out.println("Starting: "); - ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, - "decode-counts"); - stopWatch.stop(); - System.out.println("Total time: " + stopWatch.getTotalTimeSeconds()); - assertThat(cr.value().equals("Count for foobar : 1")).isTrue(); - - verify(conversionDelegate).serializeOnOutbound(any(KStream.class)); - verify(conversionDelegate).deserializeOnInbound(any(Class.class), - any(KStream.class)); - } - - } - - @EnableBinding(KafkaStreamsProcessor.class) - @EnableAutoConfiguration - public static class WordCountProcessorApplication { - - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - - return input - .flatMapValues( - value -> Arrays.asList(value.toLowerCase().split("\\W+"))) - .map((key, value) -> new KeyValue<>(value, value)) - .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) - .windowedBy(TimeWindows.of(Duration.ofSeconds(5))).count(Materialized.as("foo-WordCounts-x")) - .toStream().map((key, value) -> new KeyValue<>(null, - "Count for " + key.key() + " : " + value)); - } - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsStateStoreIntegrationTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsStateStoreIntegrationTests.java index 2cc15053e..22f0bf06d 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsStateStoreIntegrationTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsStateStoreIntegrationTests.java @@ -18,6 +18,8 @@ package org.springframework.cloud.stream.binder.kafka.streams.integration; import java.time.Duration; import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Consumer; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.kstream.KStream; @@ -32,11 +34,6 @@ import org.junit.Test; import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsStateStore; -import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsStateStoreProperties; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.kafka.core.DefaultKafkaProducerFactory; @@ -67,6 +64,7 @@ public class KafkaStreamsStateStoreIntegrationTests { app.setWebApplicationType(WebApplicationType.NONE); ConfigurableApplicationContext context = app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input", "--spring.cloud.stream.bindings.input.destination=foobar", "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" @@ -89,41 +87,14 @@ public class KafkaStreamsStateStoreIntegrationTests { } } - @Test - public void testKstreamStateStoreBuilderBeansDefinedInApplication() throws Exception { - SpringApplication app = new SpringApplication(StateStoreBeanApplication.class); - app.setWebApplicationType(WebApplicationType.NONE); - ConfigurableApplicationContext context = app.run("--server.port=0", - "--spring.jmx.enabled=false", - "--spring.cloud.stream.bindings.input3.destination=foobar", - "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", - "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" - + "=org.apache.kafka.common.serialization.Serdes$StringSerde", - "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" - + "=org.apache.kafka.common.serialization.Serdes$StringSerde", - "--spring.cloud.stream.kafka.streams.bindings.input3.consumer.applicationId" - + "=KafkaStreamsStateStoreIntegrationTests-xyzabc-123", - "--spring.cloud.stream.kafka.streams.binder.brokers=" - + embeddedKafka.getBrokersAsString()); - try { - Thread.sleep(2000); - receiveAndValidateFoo(context, StateStoreBeanApplication.class); - } - catch (Exception e) { - throw e; - } - finally { - context.close(); - } - } - - @Test public void testSameStateStoreIsCreatedOnlyOnceWhenMultipleInputBindingsArePresent() throws Exception { SpringApplication app = new SpringApplication(ProductCountApplicationWithMultipleInputBindings.class); app.setWebApplicationType(WebApplicationType.NONE); ConfigurableApplicationContext context = app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input1", + "--spring.cloud.stream.function.bindings.process-in-1=input2", "--spring.cloud.stream.bindings.input1.destination=foobar", "--spring.cloud.stream.bindings.input2.destination=hello-foobar", "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", @@ -171,22 +142,12 @@ public class KafkaStreamsStateStoreIntegrationTests { assertThat(state.persistent()).isTrue(); assertThat(productCount.processed).isTrue(); } - else if (clazz.isAssignableFrom(StateStoreBeanApplication.class)) { - StateStoreBeanApplication productCount = context - .getBean(StateStoreBeanApplication.class); - WindowStore state = productCount.state; - assertThat(state != null).isTrue(); - assertThat(state.name()).isEqualTo("mystate"); - assertThat(state.persistent()).isTrue(); - assertThat(productCount.processed).isTrue(); - } else { - fail("Expected assertiond did not happen"); + fail("Expected assertions did not happen"); } } - @EnableBinding(KafkaStreamsProcessorX.class) @EnableAutoConfiguration public static class ProductCountApplication { @@ -194,46 +155,10 @@ public class KafkaStreamsStateStoreIntegrationTests { boolean processed; - @StreamListener("input") - @KafkaStreamsStateStore(name = "mystate", type = KafkaStreamsStateStoreProperties.StoreType.WINDOW, lengthMs = 300000, retentionMs = 300000) - @SuppressWarnings({ "deprecation", "unchecked" }) - public void process(KStream input) { + @Bean + public Consumer> process() { - input.process(() -> new Processor() { - - @Override - public void init(ProcessorContext processorContext) { - state = (WindowStore) processorContext.getStateStore("mystate"); - } - - @Override - public void process(Object s, Product product) { - processed = true; - } - - @Override - public void close() { - if (state != null) { - state.close(); - } - } - }, "mystate"); - } - } - - @EnableBinding(KafkaStreamsProcessorZ.class) - @EnableAutoConfiguration - public static class StateStoreBeanApplication { - - WindowStore state; - - boolean processed; - - @StreamListener("input3") - @SuppressWarnings({"unchecked" }) - public void process(KStream input) { - - input.process(() -> new Processor() { + return input -> input.process(() -> new Processor() { @Override public void init(ProcessorContext processorContext) { @@ -263,8 +188,6 @@ public class KafkaStreamsStateStoreIntegrationTests { } } - - @EnableBinding(KafkaStreamsProcessorY.class) @EnableAutoConfiguration public static class ProductCountApplicationWithMultipleInputBindings { @@ -272,33 +195,41 @@ public class KafkaStreamsStateStoreIntegrationTests { boolean processed; - @StreamListener - @KafkaStreamsStateStore(name = "mystate", type = KafkaStreamsStateStoreProperties.StoreType.WINDOW, lengthMs = 300000, retentionMs = 300000) - @SuppressWarnings({ "deprecation", "unchecked" }) - public void process(@Input("input1")KStream input, @Input("input2")KStream input2) { + @Bean + public BiConsumer, KStream> process() { - input.process(() -> new Processor() { + return (input, input2) -> { - @Override - public void init(ProcessorContext processorContext) { - state = (WindowStore) processorContext.getStateStore("mystate"); - } + input.process(() -> new Processor() { - @Override - public void process(Object s, Product product) { - processed = true; - } - - @Override - public void close() { - if (state != null) { - state.close(); + @Override + public void init(ProcessorContext processorContext) { + state = (WindowStore) processorContext.getStateStore("mystate"); } - } - }, "mystate"); - //simple use of input2, we are not using input2 for anything other than triggering some test behavior. - input2.foreach((key, value) -> { }); + @Override + public void process(Object s, Product product) { + processed = true; + } + + @Override + public void close() { + if (state != null) { + state.close(); + } + } + }, "mystate"); + //simple use of input2, we are not using input2 for anything other than triggering some test behavior. + input2.foreach((key, value) -> { }); + }; + } + + @Bean + public StoreBuilder mystore() { + return Stores.windowStoreBuilder( + Stores.persistentWindowStore("mystate", + Duration.ofMillis(3), Duration.ofMillis(3), false), Serdes.String(), + Serdes.String()); } } @@ -315,25 +246,4 @@ public class KafkaStreamsStateStoreIntegrationTests { } } - - interface KafkaStreamsProcessorX { - - @Input("input") - KStream input(); - } - - interface KafkaStreamsProcessorY { - - @Input("input1") - KStream input1(); - - @Input("input2") - KStream input2(); - } - - interface KafkaStreamsProcessorZ { - - @Input("input3") - KStream input3(); - } } diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkastreamsBinderPojoInputStringOutputIntegrationTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkastreamsBinderPojoInputStringOutputIntegrationTests.java index 2ff80f252..130db7be4 100644 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkastreamsBinderPojoInputStringOutputIntegrationTests.java +++ b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkastreamsBinderPojoInputStringOutputIntegrationTests.java @@ -18,6 +18,7 @@ package org.springframework.cloud.stream.binder.kafka.streams.integration; import java.time.Duration; import java.util.Map; +import java.util.function.Function; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -35,10 +36,8 @@ import org.junit.Test; import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; import org.springframework.integration.test.util.TestUtils; import org.springframework.kafka.config.StreamsBuilderFactoryBean; import org.springframework.kafka.core.CleanupConfig; @@ -49,7 +48,6 @@ import org.springframework.kafka.support.serializer.JsonSerde; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.handler.annotation.SendTo; import static org.assertj.core.api.Assertions.assertThat; @@ -91,6 +89,8 @@ public class KafkastreamsBinderPojoInputStringOutputIntegrationTests { app.setWebApplicationType(WebApplicationType.NONE); ConfigurableApplicationContext context = app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.bindings.process-in-0=input", + "--spring.cloud.stream.function.bindings.process-out-0=output", "--spring.cloud.stream.bindings.input.destination=foos", "--spring.cloud.stream.bindings.output.destination=counts-id", "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", @@ -105,7 +105,7 @@ public class KafkastreamsBinderPojoInputStringOutputIntegrationTests { receiveAndValidateFoo(); // Assertions on StreamBuilderFactoryBean StreamsBuilderFactoryBean streamsBuilderFactoryBean = context - .getBean("&stream-builder-ProductCountApplication-process", StreamsBuilderFactoryBean.class); + .getBean("&stream-builder-process", StreamsBuilderFactoryBean.class); CleanupConfig cleanup = TestUtils.getPropertyValue(streamsBuilderFactoryBean, "cleanupConfig", CleanupConfig.class); assertThat(cleanup.cleanupOnStart()).isFalse(); @@ -128,15 +128,12 @@ public class KafkastreamsBinderPojoInputStringOutputIntegrationTests { assertThat(cr.value().contains("Count for product with ID 123: 1")).isTrue(); } - @EnableBinding(KafkaStreamsProcessor.class) @EnableAutoConfiguration public static class ProductCountApplication { - @StreamListener("input") - @SendTo("output") - public KStream process(KStream input) { - - return input.filter((key, product) -> product.getId() == 123) + @Bean + public Function, KStream> process() { + return input -> input.filter((key, product) -> product.getId() == 123) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Grouped.with(new JsonSerde<>(Product.class), new JsonSerde<>(Product.class))) diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/MultiProcessorsWithSameNameAndBindingTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/MultiProcessorsWithSameNameAndBindingTests.java deleted file mode 100644 index 699cee4a7..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/MultiProcessorsWithSameNameAndBindingTests.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2019-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.integration; - -import org.apache.kafka.streams.kstream.KStream; -import org.junit.ClassRule; -import org.junit.Test; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.WebApplicationType; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.kafka.config.StreamsBuilderFactoryBean; -import org.springframework.kafka.test.EmbeddedKafkaBroker; -import org.springframework.kafka.test.rule.EmbeddedKafkaRule; -import org.springframework.stereotype.Component; - -import static org.assertj.core.api.Assertions.assertThat; - -public class MultiProcessorsWithSameNameAndBindingTests { - - @ClassRule - public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, - "counts"); - - private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule - .getEmbeddedKafka(); - - @Test - public void testBinderStartsSuccessfullyWhenTwoProcessorsWithSameNamesAndBindingsPresent() { - SpringApplication app = new SpringApplication( - MultiProcessorsWithSameNameAndBindingTests.WordCountProcessorApplication.class); - app.setWebApplicationType(WebApplicationType.NONE); - - try (ConfigurableApplicationContext context = app.run("--server.port=0", - "--spring.jmx.enabled=false", - "--spring.cloud.stream.bindings.input.destination=words", - "--spring.cloud.stream.bindings.input-1.destination=words", - "--spring.cloud.stream.bindings.output.destination=counts", - "--spring.cloud.stream.bindings.output.contentType=application/json", - "--spring.cloud.stream.kafka.streams.binder.brokers=" - + embeddedKafka.getBrokersAsString())) { - StreamsBuilderFactoryBean streamsBuilderFactoryBean1 = context - .getBean("&stream-builder-Foo-process", StreamsBuilderFactoryBean.class); - assertThat(streamsBuilderFactoryBean1).isNotNull(); - StreamsBuilderFactoryBean streamsBuilderFactoryBean2 = context - .getBean("&stream-builder-Bar-process", StreamsBuilderFactoryBean.class); - assertThat(streamsBuilderFactoryBean2).isNotNull(); - } - } - - @EnableBinding(KafkaStreamsProcessorX.class) - @EnableAutoConfiguration - static class WordCountProcessorApplication { - - @Component - static class Foo { - @StreamListener - public void process(@Input("input-1") KStream input) { - } - } - - //Second class with a stub processor that has the same name as above ("process") - @Component - static class Bar { - @StreamListener - public void process(@Input("input-1") KStream input) { - } - } - } - - interface KafkaStreamsProcessorX { - - @Input("input-1") - KStream input1(); - - } -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/PerRecordAvroContentTypeTests.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/PerRecordAvroContentTypeTests.java deleted file mode 100644 index 59eee2591..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/PerRecordAvroContentTypeTests.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright 2017-2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.integration; - -import java.io.IOException; -import java.util.Map; -import java.util.Random; -import java.util.UUID; - -import com.example.Sensor; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.ByteArrayDeserializer; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.kstream.KStream; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.WebApplicationType; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.cloud.function.context.converter.avro.AvroSchemaMessageConverter; -import org.springframework.cloud.function.context.converter.avro.AvroSchemaServiceManagerImpl; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; -import org.springframework.cloud.stream.binder.kafka.streams.integration.utils.TestAvroSerializer; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.annotation.Bean; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.test.EmbeddedKafkaBroker; -import org.springframework.kafka.test.rule.EmbeddedKafkaRule; -import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.messaging.Message; -import org.springframework.messaging.converter.MessageConverter; -import org.springframework.messaging.handler.annotation.SendTo; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.util.MimeTypeUtils; - -import static org.assertj.core.api.Assertions.assertThat; - - -/** - * @author Soby Chacko - */ -public class PerRecordAvroContentTypeTests { - - @ClassRule - public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, - "received-sensors"); - - private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule - .getEmbeddedKafka(); - - private static Consumer consumer; - - @BeforeClass - public static void setUp() throws Exception { - Map consumerProps = KafkaTestUtils.consumerProps("avro-ct-test", - "false", embeddedKafka); - - // Receive the data as byte[] - consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - ByteArrayDeserializer.class); - - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( - consumerProps); - consumer = cf.createConsumer(); - embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "received-sensors"); - } - - @AfterClass - public static void tearDown() { - consumer.close(); - } - - @Test - public void testPerRecordAvroConentTypeAndVerifySerialization() throws Exception { - SpringApplication app = new SpringApplication(SensorCountAvroApplication.class); - app.setWebApplicationType(WebApplicationType.NONE); - - try (ConfigurableApplicationContext ignored = app.run("--server.port=0", - "--spring.jmx.enabled=false", - "--spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", - "--spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", - "--spring.cloud.stream.bindings.input.destination=sensors", - "--spring.cloud.stream.bindings.output.destination=received-sensors", - "--spring.cloud.stream.bindings.output.contentType=application/avro", - "--spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=per-record-avro-contentType-test", - "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", - "--spring.cloud.stream.kafka.streams.binder.brokers=" - + embeddedKafka.getBrokersAsString())) { - - Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); - // Use a custom avro test serializer - senderProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - TestAvroSerializer.class); - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( - senderProps); - try { - KafkaTemplate template = new KafkaTemplate<>(pf, true); - - Random random = new Random(); - Sensor sensor = new Sensor(); - sensor.setId(UUID.randomUUID().toString() + "-v1"); - sensor.setAcceleration(random.nextFloat() * 10); - sensor.setVelocity(random.nextFloat() * 100); - sensor.setTemperature(random.nextFloat() * 50); - // Send with avro content type set. - Message message = MessageBuilder.withPayload(sensor) - .setHeader("contentType", "application/avro").build(); - template.setDefaultTopic("sensors"); - template.send(message); - - // Serialized byte[] ^^ is received by the binding process and deserialzed - // it using avro converter. - // Then finally, the data will be output to a return topic as byte[] - // (using the same avro converter). - - // Receive the byte[] from return topic - ConsumerRecord cr = KafkaTestUtils - .getSingleRecord(consumer, "received-sensors"); - final byte[] value = cr.value(); - - // Convert the byte[] received back to avro object and verify that it is - // the same as the one we sent ^^. - AvroSchemaMessageConverter avroSchemaMessageConverter = new AvroSchemaMessageConverter(new AvroSchemaServiceManagerImpl()); - - Message receivedMessage = MessageBuilder.withPayload(value) - .setHeader("contentType", - MimeTypeUtils.parseMimeType("application/avro")) - .build(); - Sensor messageConverted = (Sensor) avroSchemaMessageConverter - .fromMessage(receivedMessage, Sensor.class); - assertThat(messageConverted).isEqualTo(sensor); - } - finally { - pf.destroy(); - } - } - } - - @EnableBinding(KafkaStreamsProcessor.class) - @EnableAutoConfiguration - static class SensorCountAvroApplication { - - @StreamListener - @SendTo("output") - public KStream process(@Input("input") KStream input) { - // return the same Sensor object unchanged so that we can do test - // verifications - return input.map(KeyValue::new); - } - - @Bean - public MessageConverter sensorMessageConverter() throws IOException { - return new AvroSchemaMessageConverter(new AvroSchemaServiceManagerImpl()); - } - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/utils/TestAvroSerializer.java b/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/utils/TestAvroSerializer.java deleted file mode 100644 index 761636ea9..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/utils/TestAvroSerializer.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2018-2019 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.binder.kafka.streams.integration.utils; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.kafka.common.serialization.Serializer; - -import org.springframework.cloud.function.context.converter.avro.AvroSchemaMessageConverter; -import org.springframework.cloud.function.context.converter.avro.AvroSchemaServiceManagerImpl; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.support.MessageBuilder; - -/** - * Custom avro serializer intended to be used for testing only. - * - * @param Target type to serialize - * @author Soby Chacko - */ -public class TestAvroSerializer implements Serializer { - - public TestAvroSerializer() { - } - - @Override - public void configure(Map configs, boolean isKey) { - - } - - @Override - public byte[] serialize(String topic, S data) { - AvroSchemaMessageConverter avroSchemaMessageConverter = new AvroSchemaMessageConverter(new AvroSchemaServiceManagerImpl()); - Message message = MessageBuilder.withPayload(data).build(); - Map headers = new HashMap<>(message.getHeaders()); - headers.put(MessageHeaders.CONTENT_TYPE, "application/avro"); - MessageHeaders messageHeaders = new MessageHeaders(headers); - final Object payload = avroSchemaMessageConverter - .toMessage(message.getPayload(), messageHeaders).getPayload(); - return (byte[]) payload; - } - - @Override - public void close() { - - } - -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/resources/avro/sensor.avsc b/spring-cloud-stream-binder-kafka-streams/src/test/resources/avro/sensor.avsc deleted file mode 100644 index c0e060d3d..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/resources/avro/sensor.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "namespace" : "com.example", - "type" : "record", - "name" : "Sensor", - "fields" : [ - {"name":"id","type":"string"}, - {"name":"temperature", "type":"float", "default":0.0}, - {"name":"acceleration", "type":"float","default":0.0}, - {"name":"velocity","type":"float","default":0.0} - ] -} diff --git a/spring-cloud-stream-binder-kafka-streams/src/test/resources/org/springframework/cloud/stream/binder/kstream/integTest-1.properties b/spring-cloud-stream-binder-kafka-streams/src/test/resources/org/springframework/cloud/stream/binder/kstream/integTest-1.properties deleted file mode 100644 index 6d983a0b1..000000000 --- a/spring-cloud-stream-binder-kafka-streams/src/test/resources/org/springframework/cloud/stream/binder/kstream/integTest-1.properties +++ /dev/null @@ -1,6 +0,0 @@ -spring.cloud.stream.bindings.input.destination=DeserializationErrorHandlerByKafkaTests-In -spring.cloud.stream.bindings.output.destination=DeserializationErrorHandlerByKafkaTests-Out -spring.cloud.stream.bindings.output.contentType=application/json -spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000 -spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde=org.apache.kafka.common.serialization.Serdes$StringSerde -spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde=org.apache.kafka.common.serialization.Serdes$StringSerde diff --git a/spring-cloud-stream-binder-kafka/pom.xml b/spring-cloud-stream-binder-kafka/pom.xml index 3a21f6fea..de5b7b519 100644 --- a/spring-cloud-stream-binder-kafka/pom.xml +++ b/spring-cloud-stream-binder-kafka/pom.xml @@ -10,7 +10,7 @@ org.springframework.cloud spring-cloud-stream-binder-kafka-parent - 3.2.2-SNAPSHOT + 4.0.0-SNAPSHOT diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java index b8d03225c..c32de1134 100644 --- a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java @@ -28,7 +28,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; import org.springframework.boot.autoconfigure.kafka.KafkaProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.stream.annotation.StreamMessageConverter; import org.springframework.cloud.stream.binder.Binder; import org.springframework.cloud.stream.binder.kafka.KafkaBinderMetrics; import org.springframework.cloud.stream.binder.kafka.KafkaBindingRebalanceListener; @@ -141,7 +140,6 @@ public class KafkaBinderConfiguration { } @Bean - @StreamMessageConverter @ConditionalOnMissingBean(KafkaNullConverter.class) MessageConverter kafkaNullConverter() { return new KafkaNullConverter(); diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderActuatorTests.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderActuatorTests.java index f722dc510..ce5e1d105 100644 --- a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderActuatorTests.java +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderActuatorTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 the original author or authors. + * Copyright 2018-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,12 +18,14 @@ package org.springframework.cloud.stream.binder.kafka.integration; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.MeterBinder; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; @@ -33,19 +35,14 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.FilteredClassLoader; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.StreamListener; import org.springframework.cloud.stream.binder.Binding; -import org.springframework.cloud.stream.binder.PollableMessageSource; import org.springframework.cloud.stream.binding.BindingService; import org.springframework.cloud.stream.config.ConsumerEndpointCustomizer; import org.springframework.cloud.stream.config.ListenerContainerCustomizer; import org.springframework.cloud.stream.config.MessageSourceCustomizer; import org.springframework.cloud.stream.config.ProducerMessageHandlerCustomizer; -import org.springframework.cloud.stream.messaging.Processor; -import org.springframework.cloud.stream.messaging.Sink; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; import org.springframework.integration.kafka.inbound.KafkaMessageSource; import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler; @@ -63,13 +60,18 @@ import static org.assertj.core.api.Assertions.assertThat; * @author Oleg Zhurakousky * @author Jon Schneider * @author Gary Russell + * @author Soby Chacko * * @since 2.0 */ @RunWith(SpringRunner.class) // @checkstyle:off -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = "spring.cloud.stream.bindings.input.group=" - + KafkaBinderActuatorTests.TEST_CONSUMER_GROUP) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, + properties = { + "spring.cloud.stream.bindings.input.group=" + KafkaBinderActuatorTests.TEST_CONSUMER_GROUP, + "spring.cloud.stream.function.bindings.process-in-0=input", + "spring.cloud.stream.pollable-source=input"} +) // @checkstyle:on @DirtiesContext public class KafkaBinderActuatorTests { @@ -100,17 +102,22 @@ public class KafkaBinderActuatorTests { @Test public void testKafkaBinderMetricsExposed() { - this.kafkaTemplate.send(Sink.INPUT, null, "foo".getBytes()); + this.kafkaTemplate.send("input", null, "foo".getBytes()); this.kafkaTemplate.flush(); assertThat(this.meterRegistry.get("spring.cloud.stream.binder.kafka.offset") - .tag("group", TEST_CONSUMER_GROUP).tag("topic", Sink.INPUT).gauge() + .tag("group", TEST_CONSUMER_GROUP).tag("topic", "input").gauge() .value()).isGreaterThan(0); } @Test + @Ignore public void testKafkaBinderMetricsWhenNoMicrometer() { new ApplicationContextRunner().withUserConfiguration(KafkaMetricsTestConfig.class) + .withPropertyValues( + "spring.cloud.stream.bindings.input.group", KafkaBinderActuatorTests.TEST_CONSUMER_GROUP, + "spring.cloud.stream.function.bindings.process-in-0", "input", + "spring.cloud.stream.pollable-source", "input") .withClassLoader(new FilteredClassLoader("io.micrometer.core")) .run(context -> { assertThat(context.getBeanNamesForType(MeterRegistry.class)) @@ -148,8 +155,8 @@ public class KafkaBinderActuatorTests { }); } - @EnableBinding({ Processor.class, PMS.class }) @EnableAutoConfiguration + @Configuration public static class KafkaMetricsTestConfig { @Bean @@ -172,19 +179,18 @@ public class KafkaBinderActuatorTests { return (handler, destinationName) -> handler.setBeanName("setByCustomizer:" + destinationName); } - @StreamListener(Sink.INPUT) - public void process(@SuppressWarnings("unused") String payload) throws InterruptedException { + @Bean + public Consumer process() { // Artificial slow listener to emulate consumer lag - Thread.sleep(1000); + return s -> { + try { + Thread.sleep(1000); + } + catch (InterruptedException e) { + //no-op + } + }; } } - - public interface PMS { - - @Input - PollableMessageSource source(); - - } - } diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderExtendedPropertiesTest.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderExtendedPropertiesTest.java index db2064c0d..64b5a79b5 100644 --- a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderExtendedPropertiesTest.java +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderExtendedPropertiesTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 the original author or authors. + * Copyright 2018-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.common.TopicPartition; @@ -33,10 +34,6 @@ import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.cloud.stream.annotation.StreamListener; import org.springframework.cloud.stream.binder.Binder; import org.springframework.cloud.stream.binder.BinderFactory; import org.springframework.cloud.stream.binder.ConsumerProperties; @@ -47,10 +44,9 @@ import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerPro import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.SubscribableChannel; -import org.springframework.messaging.handler.annotation.SendTo; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.junit4.SpringRunner; @@ -62,6 +58,11 @@ import static org.assertj.core.api.Assertions.assertThat; */ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { + "spring.cloud.stream.function.definition=process;processCustom", + "spring.cloud.stream.function.bindings.process-in-0=standard-in", + "spring.cloud.stream.function.bindings.process-out-0=standard-out", + "spring.cloud.stream.function.bindings.processCustom-in-0=custom-in", + "spring.cloud.stream.function.bindings.processCustom-out-0=custom-out", "spring.cloud.stream.kafka.bindings.standard-out.producer.configuration.key.serializer=FooSerializer.class", "spring.cloud.stream.kafka.default.producer.configuration.key.serializer=BarSerializer.class", "spring.cloud.stream.kafka.default.producer.configuration.value.serializer=BarSerializer.class", @@ -167,22 +168,19 @@ public class KafkaBinderExtendedPropertiesTest { Boolean.TRUE); } - @EnableBinding(CustomBindingForExtendedPropertyTesting.class) @EnableAutoConfiguration + @Configuration public static class KafkaMetricsTestConfig { - @StreamListener("standard-in") - @SendTo("standard-out") - public String process(String payload) { - return payload; - } - - @StreamListener("custom-in") - @SendTo("custom-out") - public String processCustom(String payload) { - return payload; + @Bean + public Function process() { + return payload -> payload; } + @Bean + public Function processCustom() { + return payload -> payload; + } @Bean public RebalanceListener rebalanceListener() { return new RebalanceListener(); @@ -190,22 +188,6 @@ public class KafkaBinderExtendedPropertiesTest { } - interface CustomBindingForExtendedPropertyTesting { - - @Input("standard-in") - SubscribableChannel standardIn(); - - @Output("standard-out") - MessageChannel standardOut(); - - @Input("custom-in") - SubscribableChannel customIn(); - - @Output("custom-out") - MessageChannel customOut(); - - } - public static class RebalanceListener implements KafkaBindingRebalanceListener { private final Map bindings = new HashMap<>(); @@ -215,23 +197,18 @@ public class KafkaBinderExtendedPropertiesTest { @Override public void onPartitionsRevokedBeforeCommit(String bindingName, Consumer consumer, Collection partitions) { - } @Override public void onPartitionsRevokedAfterCommit(String bindingName, Consumer consumer, Collection partitions) { - } @Override public void onPartitionsAssigned(String bindingName, Consumer consumer, Collection partitions, boolean initial) { - this.bindings.put(bindingName, initial); this.latch.countDown(); } - } - } diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaNullConverterTest.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaNullConverterTest.java index 3015b10f8..f796069e5 100644 --- a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaNullConverterTest.java +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaNullConverterTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2017 the original author or authors. + * Copyright 2016-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,21 +18,21 @@ package org.springframework.cloud.stream.binder.kafka.integration; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.support.KafkaNull; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; @@ -47,21 +47,19 @@ import static org.assertj.core.api.Assertions.assertThat; /** * @author Aldo Sinanaj * @author Gary Russell + * @author Soby Chacko */ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { - "spring.kafka.consumer.auto-offset-reset=earliest" }) + "spring.kafka.consumer.auto-offset-reset=earliest", + "spring.cloud.stream.function.bindings.inputListen-in-0=kafkaNullInput"}) @DirtiesContext -@Ignore public class KafkaNullConverterTest { private static final String KAFKA_BROKERS_PROPERTY = "spring.kafka.bootstrap-servers"; @Autowired - private MessageChannel kafkaNullOutput; - - @Autowired - private MessageChannel kafkaNullInput; + private ApplicationContext context; @Autowired private KafkaNullConverterTestConfig config; @@ -82,7 +80,9 @@ public class KafkaNullConverterTest { @Test public void testKafkaNullConverterOutput() throws InterruptedException { - this.kafkaNullOutput.send(new GenericMessage<>(KafkaNull.INSTANCE)); + final StreamBridge streamBridge = context.getBean(StreamBridge.class); + + streamBridge.send("kafkaNullOutput", new GenericMessage<>(KafkaNull.INSTANCE)); assertThat(this.config.countDownLatchOutput.await(10, TimeUnit.SECONDS)).isTrue(); assertThat(this.config.outputPayload).isNull(); @@ -90,14 +90,17 @@ public class KafkaNullConverterTest { @Test public void testKafkaNullConverterInput() throws InterruptedException { - this.kafkaNullInput.send(new GenericMessage<>(KafkaNull.INSTANCE)); + + final MessageChannel kafkaNullInput = context.getBean("kafkaNullInput", MessageChannel.class); + + kafkaNullInput.send(new GenericMessage<>(KafkaNull.INSTANCE)); assertThat(this.config.countDownLatchInput.await(10, TimeUnit.SECONDS)).isTrue(); assertThat(this.config.inputPayload).isNull(); } - @TestConfiguration - @EnableBinding(KafkaNullTestChannels.class) + @EnableAutoConfiguration + @Configuration public static class KafkaNullConverterTestConfig { final CountDownLatch countDownLatchOutput = new CountDownLatch(1); @@ -114,22 +117,13 @@ public class KafkaNullConverterTest { countDownLatchOutput.countDown(); } - @StreamListener("kafkaNullInput") - public void inputListen(@Payload(required = false) byte[] payload) { - this.inputPayload = payload; - countDownLatchInput.countDown(); + @Bean + public Consumer inputListen() { + return in -> { + this.inputPayload = in; + countDownLatchInput.countDown(); + }; } } - - public interface KafkaNullTestChannels { - - @Input - MessageChannel kafkaNullInput(); - - @Output - MessageChannel kafkaNullOutput(); - - } - } diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/ProducerOnlyTransactionTests.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/ProducerOnlyTransactionTests.java index 41e14f26d..861a7e152 100644 --- a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/ProducerOnlyTransactionTests.java +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/ProducerOnlyTransactionTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2019 the original author or authors. + * Copyright 2019-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,11 +35,12 @@ import org.springframework.beans.factory.BeanCreationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.stream.annotation.EnableBinding; import org.springframework.cloud.stream.binder.BinderFactory; import org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder; -import org.springframework.cloud.stream.messaging.Source; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.kafka.core.ProducerFactory; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; @@ -58,6 +59,7 @@ import static org.assertj.core.api.Assertions.assertThat; /** * @author Gary Russell + * @author Soby Chacko * @since 2.1.4 * */ @@ -80,7 +82,7 @@ public class ProducerOnlyTransactionTests { private Sender sender; @Autowired - private MessageChannel output; + private ApplicationContext context; @BeforeClass public static void setup() { @@ -95,7 +97,8 @@ public class ProducerOnlyTransactionTests { @Test public void testProducerTx() { - this.sender.DoInTransaction(this.output); + final StreamBridge streamBridge = context.getBean(StreamBridge.class); + this.sender.DoInTransaction(streamBridge); assertThat(this.sender.isInTx()).isTrue(); Map props = KafkaTestUtils.consumerProps("consumeTx", "false", embeddedKafka.getEmbeddedKafka()); @@ -109,9 +112,9 @@ public class ProducerOnlyTransactionTests { assertThat(record.value()).isEqualTo("foo".getBytes()); } - @EnableBinding(Source.class) @EnableAutoConfiguration @EnableTransactionManagement + @Configuration public static class Config { @Bean @@ -140,9 +143,9 @@ public class ProducerOnlyTransactionTests { private boolean isInTx; @Transactional - public void DoInTransaction(MessageChannel output) { + public void DoInTransaction(StreamBridge streamBridge) { this.isInTx = TransactionSynchronizationManager.isActualTransactionActive(); - output.send(new GenericMessage<>("foo")); + streamBridge.send("output", new GenericMessage<>("foo".getBytes())); } public boolean isInTx() { diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/BaseKafkaBinderTopicPropertiesUpdateTest.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/BaseKafkaBinderTopicPropertiesUpdateTest.java index bfaee22c1..72bec2af8 100644 --- a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/BaseKafkaBinderTopicPropertiesUpdateTest.java +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/BaseKafkaBinderTopicPropertiesUpdateTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 the original author or authors. + * Copyright 2018-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ package org.springframework.cloud.stream.binder.kafka.integration.topic.configs; +import java.util.function.Function; + import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -23,24 +25,21 @@ import org.junit.runner.RunWith; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.context.annotation.Bean; import org.springframework.kafka.test.rule.EmbeddedKafkaRule; -import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.SubscribableChannel; -import org.springframework.messaging.handler.annotation.SendTo; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.junit4.SpringRunner; /** * @author Heiko Does + * @author Soby Chacko */ @RunWith(SpringRunner.class) @SpringBootTest( classes = BaseKafkaBinderTopicPropertiesUpdateTest.TopicAutoConfigsTestConfig.class, webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { + "spring.cloud.stream.function.bindings.process-in-0=standard-in", + "spring.cloud.stream.function.bindings.process-out-0=standard-out", "spring.cloud.stream.kafka.bindings.standard-out.producer.topic.properties.retention.ms=9001", "spring.cloud.stream.kafka.default.producer.topic.properties.retention.ms=-1", "spring.cloud.stream.kafka.bindings.standard-in.consumer.topic.properties.retention.ms=9001", @@ -65,24 +64,12 @@ public abstract class BaseKafkaBinderTopicPropertiesUpdateTest { System.clearProperty(KAFKA_BROKERS_PROPERTY); } - @EnableBinding(CustomBindingForTopicPropertiesUpdateTesting.class) @EnableAutoConfiguration public static class TopicAutoConfigsTestConfig { - @StreamListener("standard-in") - @SendTo("standard-out") - public String process(String payload) { - return payload; + @Bean + public Function process() { + return payload -> payload; } } - - interface CustomBindingForTopicPropertiesUpdateTesting { - - @Input("standard-in") - SubscribableChannel standardIn(); - - @Output("standard-out") - MessageChannel standardOut(); - } - } diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/ConsumerProducerTransactionTests.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/ConsumerProducerTransactionTests.java index 4f49b2061..dbf30a093 100644 --- a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/ConsumerProducerTransactionTests.java +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/ConsumerProducerTransactionTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2019 the original author or authors. + * Copyright 2019-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import kafka.server.KafkaConfig; import org.junit.AfterClass; @@ -33,13 +34,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.ApplicationRunner; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.cloud.stream.annotation.StreamListener; import org.springframework.cloud.stream.config.ListenerContainerCustomizer; -import org.springframework.cloud.stream.messaging.Processor; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.core.ProducerFactory; @@ -49,8 +47,6 @@ import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import org.springframework.kafka.test.utils.KafkaTestUtils; import org.springframework.kafka.transaction.KafkaAwareTransactionManager; import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.SubscribableChannel; -import org.springframework.messaging.support.GenericMessage; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.util.backoff.FixedBackOff; @@ -61,6 +57,7 @@ import static org.mockito.Mockito.mock; /** * @author Gary Russell + * @author Soby Chacko * @since 3.0 * */ @@ -69,6 +66,11 @@ import static org.mockito.Mockito.mock; "spring.kafka.consumer.properties.isolation.level=read_committed", "spring.kafka.consumer.enable-auto-commit=false", "spring.kafka.consumer.auto-offset-reset=earliest", + "spring.cloud.function.definition=listenIn;listenIn2", + "spring.cloud.stream.function.bindings.listenIn-in-0=input", + "spring.cloud.stream.function.bindings.listenIn-out-0=output", + "spring.cloud.stream.function.bindings.listenIn2-in-0=input2", + "spring.cloud.stream.function.bindings.listenIn2-out-0=output2", "spring.cloud.stream.bindings.input.destination=consumer.producer.txIn", "spring.cloud.stream.bindings.input.group=consumer.producer.tx", "spring.cloud.stream.bindings.input.consumer.max-attempts=1", @@ -91,6 +93,9 @@ public class ConsumerProducerTransactionTests { @Autowired private Config config; + @Autowired + private ApplicationContext context; + @BeforeClass public static void setup() { System.setProperty(KAFKA_BROKERS_PROPERTY, @@ -115,26 +120,22 @@ public class ConsumerProducerTransactionTests { public void externalTM() { assertThat(this.config.input2Container.getContainerProperties().getTransactionManager()) .isSameAs(this.config.tm); - Object handler = KafkaTestUtils.getPropertyValue(this.config.output2, "dispatcher.handlers", Set.class) + final MessageChannel output2 = context.getBean("output2", MessageChannel.class); + + Object handler = KafkaTestUtils.getPropertyValue(output2, "dispatcher.handlers", Set.class) .iterator().next(); assertThat(KafkaTestUtils.getPropertyValue(handler, "delegate.kafkaTemplate.producerFactory")) .isSameAs(this.config.pf); } - @EnableBinding(TwoProcessors.class) @EnableAutoConfiguration + @Configuration public static class Config { final List outs = new ArrayList<>(); final CountDownLatch latch = new CountDownLatch(2); - @Autowired - private MessageChannel output; - - @Autowired - MessageChannel output2; - AbstractMessageListenerContainer input2Container; ProducerFactory pf; @@ -147,16 +148,19 @@ public class ConsumerProducerTransactionTests { this.latch.countDown(); } - @StreamListener(Processor.INPUT) - public void listenIn(String in) { - this.output.send(new GenericMessage<>(in.toUpperCase())); - if (in.equals("two")) { - throw new RuntimeException("fail"); - } + @Bean + public Function listenIn() { + return in -> { + if (in.equals("two")) { + throw new RuntimeException("fail"); + } + return in.toUpperCase(); + }; } - @StreamListener("input2") - public void listenIn2(String in) { + @Bean + public Function listenIn2() { + return in -> in; } @Bean @@ -187,17 +191,6 @@ public class ConsumerProducerTransactionTests { this.tm = mock; return mock; } - } - - public interface TwoProcessors extends Processor { - - @Input - SubscribableChannel input2(); - - @Output - MessageChannel output2(); - - } - } +