diff --git a/k-binder/.gitignore b/k-binder/.gitignore new file mode 100644 index 000000000..8c6847291 --- /dev/null +++ b/k-binder/.gitignore @@ -0,0 +1,26 @@ +/application.yml +/application.properties +asciidoctor.css +*~ +.#* +*# +target/ +build/ +bin/ +_site/ +.classpath +.project +.settings +.springBeans +.DS_Store +*.sw* +*.iml +*.ipr +*.iws +.idea/* +*/.idea +.factorypath +dump.rdb +.apt_generated +artifacts +.sts4-cache diff --git a/k-binder/.jdk8 b/k-binder/.jdk8 new file mode 100644 index 000000000..e69de29bb diff --git a/k-binder/.mvn/jvm.config b/k-binder/.mvn/jvm.config new file mode 100644 index 000000000..0e7dabeff --- /dev/null +++ b/k-binder/.mvn/jvm.config @@ -0,0 +1 @@ +-Xmx1024m -XX:CICompilerCount=1 -XX:TieredStopAtLevel=1 -Djava.security.egd=file:/dev/./urandom \ No newline at end of file diff --git a/k-binder/.mvn/maven.config b/k-binder/.mvn/maven.config new file mode 100644 index 000000000..3b8cf46e1 --- /dev/null +++ b/k-binder/.mvn/maven.config @@ -0,0 +1 @@ +-DaltSnapshotDeploymentRepository=repo.spring.io::default::https://repo.spring.io/libs-snapshot-local -P spring diff --git a/k-binder/.mvn/wrapper/MavenWrapperDownloader.java b/k-binder/.mvn/wrapper/MavenWrapperDownloader.java new file mode 100644 index 000000000..b901097f2 --- /dev/null +++ b/k-binder/.mvn/wrapper/MavenWrapperDownloader.java @@ -0,0 +1,117 @@ +/* + * Copyright 2007-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.net.*; +import java.io.*; +import java.nio.channels.*; +import java.util.Properties; + +public class MavenWrapperDownloader { + + private static final String WRAPPER_VERSION = "0.5.6"; + /** + * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. + */ + private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" + + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; + + /** + * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to + * use instead of the default one. + */ + private static final String MAVEN_WRAPPER_PROPERTIES_PATH = + ".mvn/wrapper/maven-wrapper.properties"; + + /** + * Path where the maven-wrapper.jar will be saved to. + */ + private static final String MAVEN_WRAPPER_JAR_PATH = + ".mvn/wrapper/maven-wrapper.jar"; + + /** + * Name of the property which should be used to override the default download url for the wrapper. + */ + private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; + + public static void main(String args[]) { + System.out.println("- Downloader started"); + File baseDirectory = new File(args[0]); + System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); + + // If the maven-wrapper.properties exists, read it and check if it contains a custom + // wrapperUrl parameter. + File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); + String url = DEFAULT_DOWNLOAD_URL; + if(mavenWrapperPropertyFile.exists()) { + FileInputStream mavenWrapperPropertyFileInputStream = null; + try { + mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); + Properties mavenWrapperProperties = new Properties(); + mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); + url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); + } catch (IOException e) { + System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); + } finally { + try { + if(mavenWrapperPropertyFileInputStream != null) { + mavenWrapperPropertyFileInputStream.close(); + } + } catch (IOException e) { + // Ignore ... + } + } + } + System.out.println("- Downloading from: " + url); + + File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); + if(!outputFile.getParentFile().exists()) { + if(!outputFile.getParentFile().mkdirs()) { + System.out.println( + "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); + } + } + System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); + try { + downloadFileFromURL(url, outputFile); + System.out.println("Done"); + System.exit(0); + } catch (Throwable e) { + System.out.println("- Error downloading"); + e.printStackTrace(); + System.exit(1); + } + } + + private static void downloadFileFromURL(String urlString, File destination) throws Exception { + if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { + String username = System.getenv("MVNW_USERNAME"); + char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); + Authenticator.setDefault(new Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(username, password); + } + }); + } + URL website = new URL(urlString); + ReadableByteChannel rbc; + rbc = Channels.newChannel(website.openStream()); + FileOutputStream fos = new FileOutputStream(destination); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + fos.close(); + rbc.close(); + } + +} diff --git a/k-binder/.mvn/wrapper/maven-wrapper.jar b/k-binder/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 000000000..2cc7d4a55 Binary files /dev/null and b/k-binder/.mvn/wrapper/maven-wrapper.jar differ diff --git a/k-binder/.mvn/wrapper/maven-wrapper.properties b/k-binder/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 000000000..642d572ce --- /dev/null +++ b/k-binder/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,2 @@ +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar diff --git a/k-binder/.settings.xml b/k-binder/.settings.xml new file mode 100644 index 000000000..3da1ea205 --- /dev/null +++ b/k-binder/.settings.xml @@ -0,0 +1,66 @@ + + + + + repo.spring.io + ${env.CI_DEPLOY_USERNAME} + ${env.CI_DEPLOY_PASSWORD} + + + + + + spring + true + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + spring-releases + Spring Releases + https://repo.spring.io/release + + false + + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + + + diff --git a/k-binder/LICENSE b/k-binder/LICENSE new file mode 100644 index 000000000..9b259bdfc --- /dev/null +++ b/k-binder/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/k-binder/README.adoc b/k-binder/README.adoc new file mode 100644 index 000000000..41f76bb91 --- /dev/null +++ b/k-binder/README.adoc @@ -0,0 +1,190 @@ +//// +DO NOT EDIT THIS FILE. IT WAS GENERATED. +Manual changes to this file will be lost when it is generated again. +Edit the files in the src/main/asciidoc/ directory instead. +//// + + +:jdkversion: 1.8 +:github-tag: master +:github-repo: spring-cloud/spring-cloud-stream-binder-kafka + +:github-raw: https://raw.githubusercontent.com/{github-repo}/{github-tag} +:github-code: https://github.com/{github-repo}/tree/{github-tag} + +image::https://circleci.com/gh/spring-cloud/spring-cloud-stream-binder-kafka.svg?style=svg["CircleCI", link="https://circleci.com/gh/spring-cloud/spring-cloud-stream-binder-kafka"] +image::https://codecov.io/gh/spring-cloud/spring-cloud-stream-binder-kafka/branch/{github-tag}/graph/badge.svg["codecov", link="https://codecov.io/gh/spring-cloud/spring-cloud-stream-binder-kafka"] +image::https://badges.gitter.im/spring-cloud/spring-cloud-stream.svg[Gitter, link="https://gitter.im/spring-cloud/spring-cloud-stream?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge"] + +// ====================================================================================== + +//= Overview +[partintro] +-- +This guide describes the Apache Kafka implementation of the Spring Cloud Stream Binder. +It contains information about its design, usage, and configuration options, as well as information on how the Stream Cloud Stream concepts map onto Apache Kafka specific constructs. +In addition, this guide explains the Kafka Streams binding capabilities of Spring Cloud Stream. +-- + +== Apache Kafka Binder + +=== Usage + +To use Apache Kafka binder, you need to add `spring-cloud-stream-binder-kafka` as a dependency to your Spring Cloud Stream application, as shown in the following example for Maven: + +[source,xml] +---- + + org.springframework.cloud + spring-cloud-stream-binder-kafka + +---- + +Alternatively, you can also use the Spring Cloud Stream Kafka Starter, as shown in the following example for Maven: + +[source,xml] +---- + + org.springframework.cloud + spring-cloud-starter-stream-kafka + +---- + +== Apache Kafka Streams Binder + +=== Usage + +To use Apache Kafka Streams binder, you need to add `spring-cloud-stream-binder-kafka-streams` as a dependency to your Spring Cloud Stream application, as shown in the following example for Maven: + +[source,xml] +---- + + org.springframework.cloud + spring-cloud-stream-binder-kafka-streams + +---- + += Appendices +[appendix] +[[building]] +== Building + +:jdkversion: 1.7 + +=== Basic Compile and Test + +To build the source you will need to install JDK {jdkversion}. + +The build uses the Maven wrapper so you don't have to install a specific +version of Maven. To enable the tests, you should have Kafka server 0.9 or above running +before building. See below for more information on running the servers. + +The main build command is + +---- +$ ./mvnw clean install +---- + +You can also add '-DskipTests' if you like, to avoid running the tests. + +NOTE: You can also install Maven (>=3.3.3) yourself and run the `mvn` command +in place of `./mvnw` in the examples below. If you do that you also +might need to add `-P spring` if your local Maven settings do not +contain repository declarations for spring pre-release artifacts. + +NOTE: Be aware that you might need to increase the amount of memory +available to Maven by setting a `MAVEN_OPTS` environment variable with +a value like `-Xmx512m -XX:MaxPermSize=128m`. We try to cover this in +the `.mvn` configuration, so if you find you have to do it to make a +build succeed, please raise a ticket to get the settings added to +source control. + + +The projects that require middleware generally include a +`docker-compose.yml`, so consider using +https://compose.docker.io/[Docker Compose] to run the middeware servers +in Docker containers. + +=== Documentation + +There is a "full" profile that will generate documentation. + +=== Working with the code +If you don't have an IDE preference we would recommend that you use +https://www.springsource.com/developer/sts[Spring Tools Suite] or +https://eclipse.org[Eclipse] when working with the code. We use the +https://eclipse.org/m2e/[m2eclipe] eclipse plugin for maven support. Other IDEs and tools +should also work without issue. + +==== Importing into eclipse with m2eclipse +We recommend the https://eclipse.org/m2e/[m2eclipe] eclipse plugin when working with +eclipse. If you don't already have m2eclipse installed it is available from the "eclipse +marketplace". + +Unfortunately m2e does not yet support Maven 3.3, so once the projects +are imported into Eclipse you will also need to tell m2eclipse to use +the `.settings.xml` file for the projects. If you do not do this you +may see many different errors related to the POMs in the +projects. Open your Eclipse preferences, expand the Maven +preferences, and select User Settings. In the User Settings field +click Browse and navigate to the Spring Cloud project you imported +selecting the `.settings.xml` file in that project. Click Apply and +then OK to save the preference changes. + +NOTE: Alternatively you can copy the repository settings from https://github.com/spring-cloud/spring-cloud-build/blob/master/.settings.xml[`.settings.xml`] into your own `~/.m2/settings.xml`. + +==== Importing into eclipse without m2eclipse +If you prefer not to use m2eclipse you can generate eclipse project metadata using the +following command: + +[indent=0] +---- + $ ./mvnw eclipse:eclipse +---- + +The generated eclipse projects can be imported by selecting `import existing projects` +from the `file` menu. +[[contributing] +== Contributing + +Spring Cloud is released under the non-restrictive Apache 2.0 license, +and follows a very standard Github development process, using Github +tracker for issues and merging pull requests into master. If you want +to contribute even something trivial please do not hesitate, but +follow the guidelines below. + +=== Sign the Contributor License Agreement +Before we accept a non-trivial patch or pull request we will need you to sign the +https://support.springsource.com/spring_committer_signup[contributor's agreement]. +Signing the contributor's agreement does not grant anyone commit rights to the main +repository, but it does mean that we can accept your contributions, and you will get an +author credit if we do. Active contributors might be asked to join the core team, and +given the ability to merge pull requests. + +=== Code Conventions and Housekeeping +None of these is essential for a pull request, but they will all help. They can also be +added after the original pull request but before a merge. + +* Use the Spring Framework code format conventions. If you use Eclipse + you can import formatter settings using the + `eclipse-code-formatter.xml` file from the + https://github.com/spring-cloud/build/tree/master/eclipse-coding-conventions.xml[Spring + Cloud Build] project. If using IntelliJ, you can use the + https://plugins.jetbrains.com/plugin/6546[Eclipse Code Formatter + Plugin] to import the same file. +* Make sure all new `.java` files to have a simple Javadoc class comment with at least an + `@author` tag identifying you, and preferably at least a paragraph on what the class is + for. +* Add the ASF license header comment to all new `.java` files (copy from existing files + in the project) +* Add yourself as an `@author` to the .java files that you modify substantially (more + than cosmetic changes). +* Add some Javadocs and, if you change the namespace, some XSD doc elements. +* A few unit tests would help a lot as well -- someone has to do it. +* If no-one else is using your branch, please rebase it against the current master (or + other target branch in the main project). +* When writing a commit message please follow https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html[these conventions], + if you are fixing an existing issue please add `Fixes gh-XXXX` at the end of the commit + message (where XXXX is the issue number). + +// ====================================================================================== diff --git a/k-binder/docs/.jdk8 b/k-binder/docs/.jdk8 new file mode 100644 index 000000000..e69de29bb diff --git a/k-binder/docs/pom.xml b/k-binder/docs/pom.xml new file mode 100644 index 000000000..982dd4bb7 --- /dev/null +++ b/k-binder/docs/pom.xml @@ -0,0 +1,65 @@ + + + 4.0.0 + spring-cloud-stream-binder-kafka-docs + + org.springframework.cloud + spring-cloud-stream-binder-kafka-parent + 3.2.3-SNAPSHOT + + jar + spring-cloud-stream-binder-kafka-docs + Spring Cloud Stream Kafka Binder Docs + + spring-cloud-stream-binder-kafka + ${basedir}/.. + 3.4 + .*stream.* + deploy + + + + ${project.groupId} + spring-cloud-starter-stream-kafka + ${project.version} + + + + src/main/asciidoc + + + + docs + + + + pl.project13.maven + git-commit-id-plugin + + + maven-dependency-plugin + + + maven-resources-plugin + + + org.codehaus.mojo + exec-maven-plugin + + + org.asciidoctor + asciidoctor-maven-plugin + + + maven-antrun-plugin + + + maven-deploy-plugin + + + + + + diff --git a/k-binder/docs/src/main/asciidoc/.gitignore b/k-binder/docs/src/main/asciidoc/.gitignore new file mode 100644 index 000000000..bbc341117 --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/.gitignore @@ -0,0 +1,2 @@ +*.html +*.css diff --git a/k-binder/docs/src/main/asciidoc/Guardfile b/k-binder/docs/src/main/asciidoc/Guardfile new file mode 100644 index 000000000..bdd4d7298 --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/Guardfile @@ -0,0 +1,20 @@ +require 'asciidoctor' +require 'erb' + +guard 'shell' do + watch(/.*\.adoc$/) {|m| + Asciidoctor.render_file('index.adoc', \ + :in_place => true, \ + :safe => Asciidoctor::SafeMode::UNSAFE, \ + :attributes=> { \ + 'source-highlighter' => 'prettify', \ + 'icons' => 'font', \ + 'linkcss'=> 'true', \ + 'copycss' => 'true', \ + 'doctype' => 'book'}) + } +end + +guard 'livereload' do + watch(%r{^.+\.(css|js|html)$}) +end diff --git a/k-binder/docs/src/main/asciidoc/README.adoc b/k-binder/docs/src/main/asciidoc/README.adoc new file mode 100644 index 000000000..ae836620e --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/README.adoc @@ -0,0 +1,57 @@ +:jdkversion: 1.8 +:github-tag: master +:github-repo: spring-cloud/spring-cloud-stream-binder-kafka + +:github-raw: https://raw.githubusercontent.com/{github-repo}/{github-tag} +:github-code: https://github.com/{github-repo}/tree/{github-tag} + +image::https://circleci.com/gh/spring-cloud/spring-cloud-stream-binder-kafka.svg?style=svg["CircleCI", link="https://circleci.com/gh/spring-cloud/spring-cloud-stream-binder-kafka"] +image::https://codecov.io/gh/spring-cloud/spring-cloud-stream-binder-kafka/branch/{github-tag}/graph/badge.svg["codecov", link="https://codecov.io/gh/spring-cloud/spring-cloud-stream-binder-kafka"] +image::https://badges.gitter.im/spring-cloud/spring-cloud-stream-binder-kafka.svg[Gitter, link="https://gitter.im/spring-cloud/spring-cloud-stream-binder-kafka?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge"] + +// ====================================================================================== + +== Apache Kafka Binder + +=== Usage + +To use Apache Kafka binder, you need to add `spring-cloud-stream-binder-kafka` as a dependency to your Spring Cloud Stream application, as shown in the following example for Maven: + +[source,xml] +---- + + org.springframework.cloud + spring-cloud-stream-binder-kafka + +---- + +Alternatively, you can also use the Spring Cloud Stream Kafka Starter, as shown in the following example for Maven: + +[source,xml] +---- + + org.springframework.cloud + spring-cloud-starter-stream-kafka + +---- + +== Apache Kafka Streams Binder + +=== Usage + +To use Apache Kafka Streams binder, you need to add `spring-cloud-stream-binder-kafka-streams` as a dependency to your Spring Cloud Stream application, as shown in the following example for Maven: + +[source,xml] +---- + + org.springframework.cloud + spring-cloud-stream-binder-kafka-streams + +---- + += Appendices +[appendix] +include::building.adoc[] +include::contributing.adoc[] + +// ====================================================================================== diff --git a/k-binder/docs/src/main/asciidoc/_configprops.adoc b/k-binder/docs/src/main/asciidoc/_configprops.adoc new file mode 100644 index 000000000..7ff5a9f02 --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/_configprops.adoc @@ -0,0 +1,68 @@ +|=== +|Name | Default | Description + +|spring.cloud.stream.binders | | Additional per-binder properties (see {@link BinderProperties}) if more then one binder of the same type is used (i.e., connect to multiple instances of RabbitMq). Here you can specify multiple binder configurations, each with different environment settings. For example; spring.cloud.stream.binders.rabbit1.environment. . . , spring.cloud.stream.binders.rabbit2.environment. . . +|spring.cloud.stream.binding-retry-interval | `30` | Retry interval (in seconds) used to schedule binding attempts. Default: 30 sec. +|spring.cloud.stream.bindings | | Additional binding properties (see {@link BinderProperties}) per binding name (e.g., 'input`). For example; This sets the content-type for the 'input' binding of a Sink application: 'spring.cloud.stream.bindings.input.contentType=text/plain' +|spring.cloud.stream.default-binder | | The name of the binder to use by all bindings in the event multiple binders available (e.g., 'rabbit'). +|spring.cloud.stream.dynamic-destination-cache-size | `10` | The maximum size of Least Recently Used (LRU) cache of dynamic destinations. Once this size is reached, new destinations will trigger the removal of old destinations. Default: 10 +|spring.cloud.stream.dynamic-destinations | `[]` | A list of destinations that can be bound dynamically. If set, only listed destinations can be bound. +|spring.cloud.stream.function.batch-mode | `false` | +|spring.cloud.stream.function.bindings | | +|spring.cloud.stream.function.definition | | Definition of functions to bind. If several functions need to be composed into one, use pipes (e.g., 'fooFunc\|barFunc') +|spring.cloud.stream.instance-count | `1` | The number of deployed instances of an application. Default: 1. NOTE: Could also be managed per individual binding "spring.cloud.stream.bindings.foo.consumer.instance-count" where 'foo' is the name of the binding. +|spring.cloud.stream.instance-index | `0` | The instance id of the application: a number from 0 to instanceCount-1. Used for partitioning and with Kafka. NOTE: Could also be managed per individual binding "spring.cloud.stream.bindings.foo.consumer.instance-index" where 'foo' is the name of the binding. +|spring.cloud.stream.instance-index-list | | A list of instance id's from 0 to instanceCount-1. Used for partitioning and with Kafka. NOTE: Could also be managed per individual binding "spring.cloud.stream.bindings.foo.consumer.instance-index-list" where 'foo' is the name of the binding. This setting will override the one set in 'spring.cloud.stream.instance-index' +|spring.cloud.stream.integration.message-handler-not-propagated-headers | | Message header names that will NOT be copied from the inbound message. +|spring.cloud.stream.kafka.binder.authorization-exception-retry-interval | | Time between retries after AuthorizationException is caught in the ListenerContainer; defalt is null which disables retries. For more info see: {@link org.springframework.kafka.listener.ConsumerProperties#setAuthorizationExceptionRetryInterval(java.time.Duration)} +|spring.cloud.stream.kafka.binder.auto-add-partitions | `false` | +|spring.cloud.stream.kafka.binder.auto-alter-topics | `false` | +|spring.cloud.stream.kafka.binder.auto-create-topics | `true` | +|spring.cloud.stream.kafka.binder.brokers | `[localhost]` | +|spring.cloud.stream.kafka.binder.certificate-store-directory | | When a certificate store location is given as classpath URL (classpath:), then the binder moves the resource from the classpath location inside the JAR to a location on the filesystem. If this value is set, then this location is used, otherwise, the certificate file is copied to the directory returned by java.io.tmpdir. +|spring.cloud.stream.kafka.binder.configuration | | Arbitrary kafka properties that apply to both producers and consumers. +|spring.cloud.stream.kafka.binder.consider-down-when-any-partition-has-no-leader | `false` | +|spring.cloud.stream.kafka.binder.consumer-properties | | Arbitrary kafka consumer properties. +|spring.cloud.stream.kafka.binder.header-mapper-bean-name | | The bean name of a custom header mapper to use instead of a {@link org.springframework.kafka.support.DefaultKafkaHeaderMapper}. +|spring.cloud.stream.kafka.binder.headers | `[]` | +|spring.cloud.stream.kafka.binder.health-timeout | `60` | Time to wait to get partition information in seconds; default 60. +|spring.cloud.stream.kafka.binder.jaas | | +|spring.cloud.stream.kafka.binder.min-partition-count | `1` | +|spring.cloud.stream.kafka.binder.producer-properties | | Arbitrary kafka producer properties. +|spring.cloud.stream.kafka.binder.replication-factor | `-1` | +|spring.cloud.stream.kafka.binder.required-acks | `1` | +|spring.cloud.stream.kafka.binder.transaction.producer.batch-timeout | | +|spring.cloud.stream.kafka.binder.transaction.producer.buffer-size | | +|spring.cloud.stream.kafka.binder.transaction.producer.compression-type | | +|spring.cloud.stream.kafka.binder.transaction.producer.configuration | | +|spring.cloud.stream.kafka.binder.transaction.producer.error-channel-enabled | | +|spring.cloud.stream.kafka.binder.transaction.producer.header-mode | | +|spring.cloud.stream.kafka.binder.transaction.producer.header-patterns | | +|spring.cloud.stream.kafka.binder.transaction.producer.message-key-expression | | +|spring.cloud.stream.kafka.binder.transaction.producer.partition-count | | +|spring.cloud.stream.kafka.binder.transaction.producer.partition-key-expression | | +|spring.cloud.stream.kafka.binder.transaction.producer.partition-key-extractor-name | | +|spring.cloud.stream.kafka.binder.transaction.producer.partition-selector-expression | | +|spring.cloud.stream.kafka.binder.transaction.producer.partition-selector-name | | +|spring.cloud.stream.kafka.binder.transaction.producer.required-groups | | +|spring.cloud.stream.kafka.binder.transaction.producer.sync | | +|spring.cloud.stream.kafka.binder.transaction.producer.topic | | +|spring.cloud.stream.kafka.binder.transaction.producer.use-native-encoding | | +|spring.cloud.stream.kafka.binder.transaction.transaction-id-prefix | | +|spring.cloud.stream.kafka.bindings | | +|spring.cloud.stream.metrics.export-properties | | List of properties that are going to be appended to each message. This gets populate by onApplicationEvent, once the context refreshes to avoid overhead of doing per message basis. +|spring.cloud.stream.metrics.key | | The name of the metric being emitted. Should be an unique value per application. Defaults to: ${spring.application.name:${vcap.application.name:${spring.config.name:application}}}. +|spring.cloud.stream.metrics.meter-filter | | Pattern to control the 'meters' one wants to capture. By default all 'meters' will be captured. For example, 'spring.integration.*' will only capture metric information for meters whose name starts with 'spring.integration'. +|spring.cloud.stream.metrics.properties | | Application properties that should be added to the metrics payload For example: `spring.application**`. +|spring.cloud.stream.metrics.schedule-interval | `60s` | Interval expressed as Duration for scheduling metrics snapshots publishing. Defaults to 60 seconds +|spring.cloud.stream.override-cloud-connectors | `false` | This property is only applicable when the cloud profile is active and Spring Cloud Connectors are provided with the application. If the property is false (the default), the binder detects a suitable bound service (for example, a RabbitMQ service bound in Cloud Foundry for the RabbitMQ binder) and uses it for creating connections (usually through Spring Cloud Connectors). When set to true, this property instructs binders to completely ignore the bound services and rely on Spring Boot properties (for example, relying on the spring.rabbitmq.* properties provided in the environment for the RabbitMQ binder). The typical usage of this property is to be nested in a customized environment when connecting to multiple systems. +|spring.cloud.stream.pollable-source | `none` | A semi-colon delimited list of binding names of pollable sources. Binding names follow the same naming convention as functions. For example, name '...pollable-source=foobar' will be accessible as 'foobar-iin-0'' binding +|spring.cloud.stream.poller.cron | | Cron expression value for the Cron Trigger. +|spring.cloud.stream.poller.fixed-delay | `1000` | Fixed delay for default poller. +|spring.cloud.stream.poller.initial-delay | `0` | Initial delay for periodic triggers. +|spring.cloud.stream.poller.max-messages-per-poll | `1` | Maximum messages per poll for the default poller. +|spring.cloud.stream.poller.time-unit | | The TimeUnit to apply to delay values. +|spring.cloud.stream.sendto.destination | `none` | The name of the header used to determine the name of the output destination +|spring.cloud.stream.source | | A colon delimited string representing the names of the sources based on which source bindings will be created. This is primarily to support cases where source binding may be required without providing a corresponding Supplier. (e.g., for cases where the actual source of data is outside of scope of spring-cloud-stream - HTTP -> Stream) + +|=== \ No newline at end of file diff --git a/k-binder/docs/src/main/asciidoc/appendix.adoc b/k-binder/docs/src/main/asciidoc/appendix.adoc new file mode 100644 index 000000000..70b828fdd --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/appendix.adoc @@ -0,0 +1,5 @@ +[[appendix]] += Appendices + + + diff --git a/k-binder/docs/src/main/asciidoc/building.adoc b/k-binder/docs/src/main/asciidoc/building.adoc new file mode 100644 index 000000000..1bc02a372 --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/building.adoc @@ -0,0 +1,78 @@ +[[building]] +== Building + +:jdkversion: 1.7 + +=== Basic Compile and Test + +To build the source you will need to install JDK {jdkversion}. + +The build uses the Maven wrapper so you don't have to install a specific +version of Maven. To enable the tests, you should have Kafka server 0.9 or above running +before building. See below for more information on running the servers. + +The main build command is + +---- +$ ./mvnw clean install +---- + +You can also add '-DskipTests' if you like, to avoid running the tests. + +NOTE: You can also install Maven (>=3.3.3) yourself and run the `mvn` command +in place of `./mvnw` in the examples below. If you do that you also +might need to add `-P spring` if your local Maven settings do not +contain repository declarations for spring pre-release artifacts. + +NOTE: Be aware that you might need to increase the amount of memory +available to Maven by setting a `MAVEN_OPTS` environment variable with +a value like `-Xmx512m -XX:MaxPermSize=128m`. We try to cover this in +the `.mvn` configuration, so if you find you have to do it to make a +build succeed, please raise a ticket to get the settings added to +source control. + + +The projects that require middleware generally include a +`docker-compose.yml`, so consider using +https://compose.docker.io/[Docker Compose] to run the middeware servers +in Docker containers. + +=== Documentation + +There is a "full" profile that will generate documentation. + +=== Working with the code +If you don't have an IDE preference we would recommend that you use +https://www.springsource.com/developer/sts[Spring Tools Suite] or +https://eclipse.org[Eclipse] when working with the code. We use the +https://eclipse.org/m2e/[m2eclipe] eclipse plugin for maven support. Other IDEs and tools +should also work without issue. + +==== Importing into eclipse with m2eclipse +We recommend the https://eclipse.org/m2e/[m2eclipe] eclipse plugin when working with +eclipse. If you don't already have m2eclipse installed it is available from the "eclipse +marketplace". + +Unfortunately m2e does not yet support Maven 3.3, so once the projects +are imported into Eclipse you will also need to tell m2eclipse to use +the `.settings.xml` file for the projects. If you do not do this you +may see many different errors related to the POMs in the +projects. Open your Eclipse preferences, expand the Maven +preferences, and select User Settings. In the User Settings field +click Browse and navigate to the Spring Cloud project you imported +selecting the `.settings.xml` file in that project. Click Apply and +then OK to save the preference changes. + +NOTE: Alternatively you can copy the repository settings from https://github.com/spring-cloud/spring-cloud-build/blob/master/.settings.xml[`.settings.xml`] into your own `~/.m2/settings.xml`. + +==== Importing into eclipse without m2eclipse +If you prefer not to use m2eclipse you can generate eclipse project metadata using the +following command: + +[indent=0] +---- + $ ./mvnw eclipse:eclipse +---- + +The generated eclipse projects can be imported by selecting `import existing projects` +from the `file` menu. diff --git a/k-binder/docs/src/main/asciidoc/contributing.adoc b/k-binder/docs/src/main/asciidoc/contributing.adoc new file mode 100644 index 000000000..65523be8f --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/contributing.adoc @@ -0,0 +1,42 @@ +[[contributing] +== Contributing + +Spring Cloud is released under the non-restrictive Apache 2.0 license, +and follows a very standard Github development process, using Github +tracker for issues and merging pull requests into master. If you want +to contribute even something trivial please do not hesitate, but +follow the guidelines below. + +=== Sign the Contributor License Agreement +Before we accept a non-trivial patch or pull request we will need you to sign the +https://support.springsource.com/spring_committer_signup[contributor's agreement]. +Signing the contributor's agreement does not grant anyone commit rights to the main +repository, but it does mean that we can accept your contributions, and you will get an +author credit if we do. Active contributors might be asked to join the core team, and +given the ability to merge pull requests. + +=== Code Conventions and Housekeeping +None of these is essential for a pull request, but they will all help. They can also be +added after the original pull request but before a merge. + +* Use the Spring Framework code format conventions. If you use Eclipse + you can import formatter settings using the + `eclipse-code-formatter.xml` file from the + https://github.com/spring-cloud/build/tree/master/eclipse-coding-conventions.xml[Spring + Cloud Build] project. If using IntelliJ, you can use the + https://plugins.jetbrains.com/plugin/6546[Eclipse Code Formatter + Plugin] to import the same file. +* Make sure all new `.java` files to have a simple Javadoc class comment with at least an + `@author` tag identifying you, and preferably at least a paragraph on what the class is + for. +* Add the ASF license header comment to all new `.java` files (copy from existing files + in the project) +* Add yourself as an `@author` to the .java files that you modify substantially (more + than cosmetic changes). +* Add some Javadocs and, if you change the namespace, some XSD doc elements. +* A few unit tests would help a lot as well -- someone has to do it. +* If no-one else is using your branch, please rebase it against the current master (or + other target branch in the main project). +* When writing a commit message please follow https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html[these conventions], + if you are fixing an existing issue please add `Fixes gh-XXXX` at the end of the commit + message (where XXXX is the issue number). \ No newline at end of file diff --git a/k-binder/docs/src/main/asciidoc/dlq.adoc b/k-binder/docs/src/main/asciidoc/dlq.adoc new file mode 100644 index 000000000..bccd7b404 --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/dlq.adoc @@ -0,0 +1,158 @@ +[[kafka-dlq-processing]] +=== Dead-Letter Topic Processing + +[[dlq-partition-selection]] +==== Dead-Letter Topic Partition Selection + +By default, records are published to the Dead-Letter topic using the same partition as the original record. +This means the Dead-Letter topic must have at least as many partitions as the original record. + +To change this behavior, add a `DlqPartitionFunction` implementation as a `@Bean` to the application context. +Only one such bean can be present. +The function is provided with the consumer group, the failed `ConsumerRecord` and the exception. +For example, if you always want to route to partition 0, you might use: + +==== +[source, java] +---- +@Bean +public DlqPartitionFunction partitionFunction() { + return (group, record, ex) -> 0; +} +---- +==== +NOTE: If you set a consumer binding's `dlqPartitions` property to 1 (and the binder's `minPartitionCount` is equal to `1`), there is no need to supply a `DlqPartitionFunction`; the framework will always use partition 0. +If you set a consumer binding's `dlqPartitions` property to a value greater than `1` (or the binder's `minPartitionCount` is greater than `1`), you **must** provide a `DlqPartitionFunction` bean, even if the partition count is the same as the original topic's. + +It is also possible to define a custom name for the DLQ topic. +In order to do so, create an implementation of `DlqDestinationResolver` as a `@Bean` to the application context. +When the binder detects such a bean, that takes precedence, otherwise it will use the `dlqName` property. +If neither of these are found, it will default to `error..`. +Here is an example of `DlqDestinationResolver` as a `@Bean`. + +==== +[source] +---- +@Bean +public DlqDestinationResolver dlqDestinationResolver() { + return (rec, ex) -> { + if (rec.topic().equals("word1")) { + return "topic1-dlq"; + } + else { + return "topic2-dlq"; + } + }; +} +---- +==== + +One important thing to keep in mind when providing an implementation for `DlqDestinationResolver` is that the provisioner in the binder will not auto create topics for the application. +This is because there is no way for the binder to infer the names of all the DLQ topics the implementation might send to. +Therefore, if you provide DLQ names using this strategy, it is the application's responsibility to ensure that those topics are created beforehand. + +[[dlq-handling]] +==== Handling Records in a Dead-Letter Topic + +Because the framework cannot anticipate how users would want to dispose of dead-lettered messages, it does not provide any standard mechanism to handle them. +If the reason for the dead-lettering is transient, you may wish to route the messages back to the original topic. +However, if the problem is a permanent issue, that could cause an infinite loop. +The sample Spring Boot application within this topic is an example of how to route those messages back to the original topic, but it moves them to a "`parking lot`" topic after three attempts. +The application is another spring-cloud-stream application that reads from the dead-letter topic. +It exits when no messages are received for 5 seconds. + +The examples assume the original destination is `so8400out` and the consumer group is `so8400`. + +There are a couple of strategies to consider: + +* Consider running the rerouting only when the main application is not running. +Otherwise, the retries for transient errors are used up very quickly. +* Alternatively, use a two-stage approach: Use this application to route to a third topic and another to route from there back to the main topic. + +The following code listings show the sample application: + +.application.properties +[source] +---- +spring.cloud.stream.bindings.input.group=so8400replay +spring.cloud.stream.bindings.input.destination=error.so8400out.so8400 + +spring.cloud.stream.bindings.output.destination=so8400out + +spring.cloud.stream.bindings.parkingLot.destination=so8400in.parkingLot + +spring.cloud.stream.kafka.binder.configuration.auto.offset.reset=earliest + +spring.cloud.stream.kafka.binder.headers=x-retries +---- + +.Application +[source, java] +---- +@SpringBootApplication +@EnableBinding(TwoOutputProcessor.class) +public class ReRouteDlqKApplication implements CommandLineRunner { + + private static final String X_RETRIES_HEADER = "x-retries"; + + public static void main(String[] args) { + SpringApplication.run(ReRouteDlqKApplication.class, args).close(); + } + + private final AtomicInteger processed = new AtomicInteger(); + + @Autowired + private MessageChannel parkingLot; + + @StreamListener(Processor.INPUT) + @SendTo(Processor.OUTPUT) + public Message reRoute(Message failed) { + processed.incrementAndGet(); + Integer retries = failed.getHeaders().get(X_RETRIES_HEADER, Integer.class); + if (retries == null) { + System.out.println("First retry for " + failed); + return MessageBuilder.fromMessage(failed) + .setHeader(X_RETRIES_HEADER, new Integer(1)) + .setHeader(BinderHeaders.PARTITION_OVERRIDE, + failed.getHeaders().get(KafkaHeaders.RECEIVED_PARTITION_ID)) + .build(); + } + else if (retries.intValue() < 3) { + System.out.println("Another retry for " + failed); + return MessageBuilder.fromMessage(failed) + .setHeader(X_RETRIES_HEADER, new Integer(retries.intValue() + 1)) + .setHeader(BinderHeaders.PARTITION_OVERRIDE, + failed.getHeaders().get(KafkaHeaders.RECEIVED_PARTITION_ID)) + .build(); + } + else { + System.out.println("Retries exhausted for " + failed); + parkingLot.send(MessageBuilder.fromMessage(failed) + .setHeader(BinderHeaders.PARTITION_OVERRIDE, + failed.getHeaders().get(KafkaHeaders.RECEIVED_PARTITION_ID)) + .build()); + } + return null; + } + + @Override + public void run(String... args) throws Exception { + while (true) { + int count = this.processed.get(); + Thread.sleep(5000); + if (count == this.processed.get()) { + System.out.println("Idle, exiting"); + return; + } + } + } + + public interface TwoOutputProcessor extends Processor { + + @Output("parkingLot") + MessageChannel parkingLot(); + + } + +} +---- diff --git a/k-binder/docs/src/main/asciidoc/ghpages.sh b/k-binder/docs/src/main/asciidoc/ghpages.sh new file mode 100755 index 000000000..375ce6c4a --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/ghpages.sh @@ -0,0 +1,330 @@ +#!/bin/bash -x + +set -e + +# Set default props like MAVEN_PATH, ROOT_FOLDER etc. +function set_default_props() { + # The script should be run from the root folder + ROOT_FOLDER=`pwd` + echo "Current folder is ${ROOT_FOLDER}" + + if [[ ! -e "${ROOT_FOLDER}/.git" ]]; then + echo "You're not in the root folder of the project!" + exit 1 + fi + + # Prop that will let commit the changes + COMMIT_CHANGES="no" + MAVEN_PATH=${MAVEN_PATH:-} + echo "Path to Maven is [${MAVEN_PATH}]" + REPO_NAME=${PWD##*/} + echo "Repo name is [${REPO_NAME}]" + SPRING_CLOUD_STATIC_REPO=${SPRING_CLOUD_STATIC_REPO:-git@github.com:spring-cloud/spring-cloud-static.git} + echo "Spring Cloud Static repo is [${SPRING_CLOUD_STATIC_REPO}" +} + +# Check if gh-pages exists and docs have been built +function check_if_anything_to_sync() { + git remote set-url --push origin `git config remote.origin.url | sed -e 's/^git:/https:/'` + + if ! (git remote set-branches --add origin gh-pages && git fetch -q); then + echo "No gh-pages, so not syncing" + exit 0 + fi + + if ! [ -d docs/target/generated-docs ] && ! [ "${BUILD}" == "yes" ]; then + echo "No gh-pages sources in docs/target/generated-docs, so not syncing" + exit 0 + fi +} + +function retrieve_current_branch() { + # Code getting the name of the current branch. For master we want to publish as we did until now + # https://stackoverflow.com/questions/1593051/how-to-programmatically-determine-the-current-checked-out-git-branch + # If there is a branch already passed will reuse it - otherwise will try to find it + CURRENT_BRANCH=${BRANCH} + if [[ -z "${CURRENT_BRANCH}" ]] ; then + CURRENT_BRANCH=$(git symbolic-ref -q HEAD) + CURRENT_BRANCH=${CURRENT_BRANCH##refs/heads/} + CURRENT_BRANCH=${CURRENT_BRANCH:-HEAD} + fi + echo "Current branch is [${CURRENT_BRANCH}]" + git checkout ${CURRENT_BRANCH} || echo "Failed to check the branch... continuing with the script" +} + +# Switches to the provided value of the release version. We always prefix it with `v` +function switch_to_tag() { + git checkout v${VERSION} +} + +# Build the docs if switch is on +function build_docs_if_applicable() { + if [[ "${BUILD}" == "yes" ]] ; then + ./mvnw clean install -P docs -pl docs -DskipTests + fi +} + +# Get the name of the `docs.main` property +# Get allowed branches - assumes that a `docs` module is available under `docs` profile +function retrieve_doc_properties() { + MAIN_ADOC_VALUE=$("${MAVEN_PATH}"mvn -q \ + -Dexec.executable="echo" \ + -Dexec.args='${docs.main}' \ + --non-recursive \ + org.codehaus.mojo:exec-maven-plugin:1.3.1:exec) + echo "Extracted 'main.adoc' from Maven build [${MAIN_ADOC_VALUE}]" + + + ALLOW_PROPERTY=${ALLOW_PROPERTY:-"docs.allowed.branches"} + ALLOWED_BRANCHES_VALUE=$("${MAVEN_PATH}"mvn -q \ + -Dexec.executable="echo" \ + -Dexec.args="\${${ALLOW_PROPERTY}}" \ + org.codehaus.mojo:exec-maven-plugin:1.3.1:exec \ + -P docs \ + -pl docs) + echo "Extracted '${ALLOW_PROPERTY}' from Maven build [${ALLOWED_BRANCHES_VALUE}]" +} + +# Stash any outstanding changes +function stash_changes() { + git diff-index --quiet HEAD && dirty=$? || (echo "Failed to check if the current repo is dirty. Assuming that it is." && dirty="1") + if [ "$dirty" != "0" ]; then git stash; fi +} + +# Switch to gh-pages branch to sync it with current branch +function add_docs_from_target() { + local DESTINATION_REPO_FOLDER + if [[ -z "${DESTINATION}" && -z "${CLONE}" ]] ; then + DESTINATION_REPO_FOLDER=${ROOT_FOLDER} + elif [[ "${CLONE}" == "yes" ]]; then + mkdir -p ${ROOT_FOLDER}/target + local clonedStatic=${ROOT_FOLDER}/target/spring-cloud-static + if [[ ! -e "${clonedStatic}/.git" ]]; then + echo "Cloning Spring Cloud Static to target" + git clone ${SPRING_CLOUD_STATIC_REPO} ${clonedStatic} && git checkout gh-pages + else + echo "Spring Cloud Static already cloned - will pull changes" + cd ${clonedStatic} && git checkout gh-pages && git pull origin gh-pages + fi + DESTINATION_REPO_FOLDER=${clonedStatic}/${REPO_NAME} + mkdir -p ${DESTINATION_REPO_FOLDER} + else + if [[ ! -e "${DESTINATION}/.git" ]]; then + echo "[${DESTINATION}] is not a git repository" + exit 1 + fi + DESTINATION_REPO_FOLDER=${DESTINATION}/${REPO_NAME} + mkdir -p ${DESTINATION_REPO_FOLDER} + echo "Destination was provided [${DESTINATION}]" + fi + cd ${DESTINATION_REPO_FOLDER} + git checkout gh-pages + git pull origin gh-pages + + # Add git branches + ################################################################### + if [[ -z "${VERSION}" ]] ; then + copy_docs_for_current_version + else + copy_docs_for_provided_version + fi + commit_changes_if_applicable +} + + +# Copies the docs by using the retrieved properties from Maven build +function copy_docs_for_current_version() { + if [[ "${CURRENT_BRANCH}" == "master" ]] ; then + echo -e "Current branch is master - will copy the current docs only to the root folder" + for f in docs/target/generated-docs/*; do + file=${f#docs/target/generated-docs/*} + if ! git ls-files -i -o --exclude-standard --directory | grep -q ^$file$; then + # Not ignored... + cp -rf $f ${ROOT_FOLDER}/ + git add -A ${ROOT_FOLDER}/$file + fi + done + COMMIT_CHANGES="yes" + else + echo -e "Current branch is [${CURRENT_BRANCH}]" + # https://stackoverflow.com/questions/29300806/a-bash-script-to-check-if-a-string-is-present-in-a-comma-separated-list-of-strin + if [[ ",${ALLOWED_BRANCHES_VALUE}," = *",${CURRENT_BRANCH},"* ]] ; then + mkdir -p ${ROOT_FOLDER}/${CURRENT_BRANCH} + echo -e "Branch [${CURRENT_BRANCH}] is allowed! Will copy the current docs to the [${CURRENT_BRANCH}] folder" + for f in docs/target/generated-docs/*; do + file=${f#docs/target/generated-docs/*} + if ! git ls-files -i -o --exclude-standard --directory | grep -q ^$file$; then + # Not ignored... + # We want users to access 2.0.0.BUILD-SNAPSHOT/ instead of 1.0.0.RELEASE/spring-cloud.sleuth.html + if [[ "${file}" == "${MAIN_ADOC_VALUE}.html" ]] ; then + # We don't want to copy the spring-cloud-sleuth.html + # we want it to be converted to index.html + cp -rf $f ${ROOT_FOLDER}/${CURRENT_BRANCH}/index.html + git add -A ${ROOT_FOLDER}/${CURRENT_BRANCH}/index.html + else + cp -rf $f ${ROOT_FOLDER}/${CURRENT_BRANCH} + git add -A ${ROOT_FOLDER}/${CURRENT_BRANCH}/$file + fi + fi + done + COMMIT_CHANGES="yes" + else + echo -e "Branch [${CURRENT_BRANCH}] is not on the allow list! Check out the Maven [${ALLOW_PROPERTY}] property in + [docs] module available under [docs] profile. Won't commit any changes to gh-pages for this branch." + fi + fi +} + +# Copies the docs by using the explicitly provided version +function copy_docs_for_provided_version() { + local FOLDER=${DESTINATION_REPO_FOLDER}/${VERSION} + mkdir -p ${FOLDER} + echo -e "Current tag is [v${VERSION}] Will copy the current docs to the [${FOLDER}] folder" + for f in ${ROOT_FOLDER}/docs/target/generated-docs/*; do + file=${f#${ROOT_FOLDER}/docs/target/generated-docs/*} + copy_docs_for_branch ${file} ${FOLDER} + done + COMMIT_CHANGES="yes" + CURRENT_BRANCH="v${VERSION}" +} + +# Copies the docs from target to the provided destination +# Params: +# $1 - file from target +# $2 - destination to which copy the files +function copy_docs_for_branch() { + local file=$1 + local destination=$2 + if ! git ls-files -i -o --exclude-standard --directory | grep -q ^${file}$; then + # Not ignored... + # We want users to access 2.0.0.BUILD-SNAPSHOT/ instead of 1.0.0.RELEASE/spring-cloud.sleuth.html + if [[ ("${file}" == "${MAIN_ADOC_VALUE}.html") || ("${file}" == "${REPO_NAME}.html") ]] ; then + # We don't want to copy the spring-cloud-sleuth.html + # we want it to be converted to index.html + cp -rf $f ${destination}/index.html + git add -A ${destination}/index.html + else + cp -rf $f ${destination} + git add -A ${destination}/$file + fi + fi +} + +function commit_changes_if_applicable() { + if [[ "${COMMIT_CHANGES}" == "yes" ]] ; then + COMMIT_SUCCESSFUL="no" + git commit -a -m "Sync docs from ${CURRENT_BRANCH} to gh-pages" && COMMIT_SUCCESSFUL="yes" || echo "Failed to commit changes" + + # Uncomment the following push if you want to auto push to + # the gh-pages branch whenever you commit to master locally. + # This is a little extreme. Use with care! + ################################################################### + if [[ "${COMMIT_SUCCESSFUL}" == "yes" ]] ; then + git push origin gh-pages + fi + fi +} + +# Switch back to the previous branch and exit block +function checkout_previous_branch() { + # If -version was provided we need to come back to root project + cd ${ROOT_FOLDER} + git checkout ${CURRENT_BRANCH} || echo "Failed to check the branch... continuing with the script" + if [ "$dirty" != "0" ]; then git stash pop; fi + exit 0 +} + +# Assert if properties have been properly passed +function assert_properties() { +echo "VERSION [${VERSION}], DESTINATION [${DESTINATION}], CLONE [${CLONE}]" +if [[ "${VERSION}" != "" && (-z "${DESTINATION}" && -z "${CLONE}") ]] ; then echo "Version was set but destination / clone was not!"; exit 1;fi +if [[ ("${DESTINATION}" != "" && "${CLONE}" != "") && -z "${VERSION}" ]] ; then echo "Destination / clone was set but version was not!"; exit 1;fi +if [[ "${DESTINATION}" != "" && "${CLONE}" == "yes" ]] ; then echo "Destination and clone was set. Pick one!"; exit 1;fi +} + +# Prints the usage +function print_usage() { +cat </` +- if the destination switch is passed (-d) then the script will check if the provided dir is a git repo and then will + switch to gh-pages of that repo and copy the generated docs to `docs//` + +USAGE: + +You can use the following options: + +-v|--version - the script will apply the whole procedure for a particular library version +-d|--destination - the root of destination folder where the docs should be copied. You have to use the full path. + E.g. point to spring-cloud-static folder. Can't be used with (-c) +-b|--build - will run the standard build process after checking out the branch +-c|--clone - will automatically clone the spring-cloud-static repo instead of providing the destination. + Obviously can't be used with (-d) + +EOF +} + + +# ========================================== +# ____ ____ _____ _____ _____ _______ +# / ____|/ ____| __ \|_ _| __ \__ __| +# | (___ | | | |__) | | | | |__) | | | +# \___ \| | | _ / | | | ___/ | | +# ____) | |____| | \ \ _| |_| | | | +# |_____/ \_____|_| \_\_____|_| |_| +# +# ========================================== + +while [[ $# > 0 ]] +do +key="$1" +case ${key} in + -v|--version) + VERSION="$2" + shift # past argument + ;; + -d|--destination) + DESTINATION="$2" + shift # past argument + ;; + -b|--build) + BUILD="yes" + ;; + -c|--clone) + CLONE="yes" + ;; + -h|--help) + print_usage + exit 0 + ;; + *) + echo "Invalid option: [$1]" + print_usage + exit 1 + ;; +esac +shift # past argument or value +done + +assert_properties +set_default_props +check_if_anything_to_sync +if [[ -z "${VERSION}" ]] ; then + retrieve_current_branch +else + switch_to_tag +fi +build_docs_if_applicable +retrieve_doc_properties +stash_changes +add_docs_from_target +checkout_previous_branch diff --git a/k-binder/docs/src/main/asciidoc/images/kafka-binder.png b/k-binder/docs/src/main/asciidoc/images/kafka-binder.png new file mode 100755 index 000000000..7c6c4c020 Binary files /dev/null and b/k-binder/docs/src/main/asciidoc/images/kafka-binder.png differ diff --git a/k-binder/docs/src/main/asciidoc/images/kafka-streams-initializr.png b/k-binder/docs/src/main/asciidoc/images/kafka-streams-initializr.png new file mode 100644 index 000000000..3f1014895 Binary files /dev/null and b/k-binder/docs/src/main/asciidoc/images/kafka-streams-initializr.png differ diff --git a/k-binder/docs/src/main/asciidoc/images/spring-initializr-kafka-streams.png b/k-binder/docs/src/main/asciidoc/images/spring-initializr-kafka-streams.png new file mode 100644 index 000000000..5d820fe55 Binary files /dev/null and b/k-binder/docs/src/main/asciidoc/images/spring-initializr-kafka-streams.png differ diff --git a/k-binder/docs/src/main/asciidoc/index-docinfo.xml b/k-binder/docs/src/main/asciidoc/index-docinfo.xml new file mode 100644 index 000000000..0e89c2b13 --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/index-docinfo.xml @@ -0,0 +1,14 @@ +Spring Cloud Stream Kafka Binder +{spring-cloud-stream-binder-kafka-version} + + 2013-2016 + Pivotal Software, Inc. + + + + Copies of this document may be made for your own use and for distribution to + others, provided that you do not charge any fee for such copies and further + provided that each copy contains this Copyright Notice, whether distributed in + print or electronically. + + diff --git a/k-binder/docs/src/main/asciidoc/kafka-streams.adoc b/k-binder/docs/src/main/asciidoc/kafka-streams.adoc new file mode 100644 index 000000000..138badbe8 --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/kafka-streams.adoc @@ -0,0 +1,2249 @@ +== Kafka Streams Binder + + +=== Usage + +For using the Kafka Streams binder, you just need to add it to your Spring Cloud Stream application, using the following maven coordinates: + +[source,xml] +---- + + org.springframework.cloud + spring-cloud-stream-binder-kafka-streams + +---- + +A quick way to bootstrap a new project for Kafka Streams binder is to use http://start.spring.io[Spring Initializr] and then select "Cloud Streams" and "Spring for Kafka Streams" as shown below + +image::{github-raw}/docs/src/main/asciidoc/images/spring-initializr-kafka-streams.png[width=800,scaledwidth="75%",align="center"] + +=== Overview + +Spring Cloud Stream includes a binder implementation designed explicitly for https://kafka.apache.org/documentation/streams/[Apache Kafka Streams] binding. +With this native integration, a Spring Cloud Stream "processor" application can directly use the +https://kafka.apache.org/documentation/streams/developer-guide[Apache Kafka Streams] APIs in the core business logic. + +Kafka Streams binder implementation builds on the foundations provided by the https://docs.spring.io/spring-kafka/reference/html/#kafka-streams[Spring for Apache Kafka] project. + +Kafka Streams binder provides binding capabilities for the three major types in Kafka Streams - `KStream`, `KTable` and `GlobalKTable`. + +Kafka Streams applications typically follow a model in which the records are read from an inbound topic, apply business logic, and then write the transformed records to an outbound topic. +Alternatively, a Processor application with no outbound destination can be defined as well. + +In the following sections, we are going to look at the details of Spring Cloud Stream's integration with Kafka Streams. + +=== Programming Model + +When using the programming model provided by Kafka Streams binder, both the high-level https://docs.confluent.io/current/streams/developer-guide/dsl-api.html[Streams DSL] and a mix of both the higher level and the lower level https://docs.confluent.io/current/streams/developer-guide/processor-api.html[Processor-API] can be used as options. +When mixing both higher and lower level API's, this is usually achieved by invoking `transform` or `process` API methods on `KStream`. + +==== Functional Style + +Starting with Spring Cloud Stream `3.0.0`, Kafka Streams binder allows the applications to be designed and developed using the functional programming style that is available in Java 8. +This means that the applications can be concisely represented as a lambda expression of types `java.util.function.Function` or `java.util.function.Consumer`. + +Let's take a very basic example. + +[source] +---- +@SpringBootApplication +public class SimpleConsumerApplication { + + @Bean + public java.util.function.Consumer> process() { + + return input -> + input.foreach((key, value) -> { + System.out.println("Key: " + key + " Value: " + value); + }); + } +} +---- + +Albeit simple, this is a complete standalone Spring Boot application that is leveraging Kafka Streams for stream processing. +This is a consumer application with no outbound binding and only a single inbound binding. +The application consumes data and it simply logs the information from the `KStream` key and value on the standard output. +The application contains the `SpringBootApplication` annotation and a method that is marked as `Bean`. +The bean method is of type `java.util.function.Consumer` which is parameterized with `KStream`. +Then in the implementation, we are returning a Consumer object that is essentially a lambda expression. +Inside the lambda expression, the code for processing the data is provided. + +In this application, there is a single input binding that is of type `KStream`. +The binder creates this binding for the application with a name `process-in-0`, i.e. the name of the function bean name followed by a dash character (`-`) and the literal `in` followed by another dash and then the ordinal position of the parameter. +You use this binding name to set other properties such as destination. +For example, `spring.cloud.stream.bindings.process-in-0.destination=my-topic`. + +NOTE: If the destination property is not set on the binding, a topic is created with the same name as the binding (if there are sufficient privileges for the application) or that topic is expected to be already available. + +Once built as a uber-jar (e.g., `kstream-consumer-app.jar`), you can run the above example like the following. + +If the applications choose to define the functional beans using Spring's `Component` annotation, the binder also suppports that model. +The above functional bean could be rewritten as below. + +``` +@Component(name = "process") +public class SimpleConsumer implements java.util.function.Consumer> { + + @Override + public void accept(KStream input) { + input.foreach((key, value) -> { + System.out.println("Key: " + key + " Value: " + value); + }); + } +} +``` + +[source] +---- +java -jar kstream-consumer-app.jar --spring.cloud.stream.bindings.process-in-0.destination=my-topic +---- + +Here is another example, where it is a full processor with both input and output bindings. +This is the classic word-count example in which the application receives data from a topic, the number of occurrences for each word is then computed in a tumbling time-window. + +[source] +---- +@SpringBootApplication +public class WordCountProcessorApplication { + + @Bean + public Function, KStream> process() { + + return input -> input + .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .map((key, value) -> new KeyValue<>(value, value)) + .groupByKey(Serialized.with(Serdes.String(), Serdes.String())) + .windowedBy(TimeWindows.of(5000)) + .count(Materialized.as("word-counts-state-store")) + .toStream() + .map((key, value) -> new KeyValue<>(key.key(), new WordCount(key.key(), value, + new Date(key.window().start()), new Date(key.window().end())))); + } + + public static void main(String[] args) { + SpringApplication.run(WordCountProcessorApplication.class, args); + } +} +---- + +Here again, this is a complete Spring Boot application. The difference here from the first application is that the bean method is of type `java.util.function.Function`. +The first parameterized type for the `Function` is for the input `KStream` and the second one is for the output. +In the method body, a lambda expression is provided that is of type `Function` and as implementation, the actual business logic is given. +Similar to the previously discussed Consumer based application, the input binding here is named as `process-in-0` by default. For the output, the binding name is automatically also set to `process-out-0`. + +Once built as an uber-jar (e.g., `wordcount-processor.jar`), you can run the above example like the following. + +[source] +---- +java -jar wordcount-processor.jar --spring.cloud.stream.bindings.process-in-0.destination=words --spring.cloud.stream.bindings.process-out-0.destination=counts +---- + +This application will consume messages from the Kafka topic `words` and the computed results are published to an output +topic `counts`. + +Spring Cloud Stream will ensure that the messages from both the incoming and outgoing topics are automatically bound as +KStream objects. As a developer, you can exclusively focus on the business aspects of the code, i.e. writing the logic +required in the processor. Setting up Kafka Streams specific configuration required by the Kafka Streams infrastructure +is automatically handled by the framework. + +The two examples we saw above have a single `KStream` input binding. In both cases, the bindings received the records from a single topic. +If you want to multiplex multiple topics into a single `KStream` binding, you can provide comma separated Kafka topics as destinations below. + +`spring.cloud.stream.bindings.process-in-0.destination=topic-1,topic-2,topic-3` + +In addition, you can also provide topic patterns as destinations if you want to match topics against a regular exression. + +`spring.cloud.stream.bindings.process-in-0.destination=input.*` + +===== Multiple Input Bindings + +Many non-trivial Kafka Streams applications often consume data from more than one topic through multiple bindings. +For instance, one topic is consumed as `Kstream` and another as `KTable` or `GlobalKTable`. +There are many reasons why an application might want to receive data as a table type. +Think of a use-case where the underlying topic is populated through a change data capture (CDC) mechanism from a database or perhaps the application only cares about the latest updates for downstream processing. +If the application specifies that the data needs to be bound as `KTable` or `GlobalKTable`, then Kafka Streams binder will properly bind the destination to a `KTable` or `GlobalKTable` and make them available for the application to operate upon. +We will look at a few different scenarios how multiple input bindings are handled in the Kafka Streams binder. + +====== BiFunction in Kafka Streams Binder + +Here is an example where we have two inputs and an output. In this case, the application can leverage on `java.util.function.BiFunction`. + +[source] +---- +@Bean +public BiFunction, KTable, KStream> process() { + return (userClicksStream, userRegionsTable) -> (userClicksStream + .leftJoin(userRegionsTable, (clicks, region) -> new RegionWithClicks(region == null ? + "UNKNOWN" : region, clicks), + Joined.with(Serdes.String(), Serdes.Long(), null)) + .map((user, regionWithClicks) -> new KeyValue<>(regionWithClicks.getRegion(), + regionWithClicks.getClicks())) + .groupByKey(Grouped.with(Serdes.String(), Serdes.Long())) + .reduce(Long::sum) + .toStream()); +} +---- + +Here again, the basic theme is the same as in the previous examples, but here we have two inputs. +Java's `BiFunction` support is used to bind the inputs to the desired destinations. +The default binding names generated by the binder for the inputs are `process-in-0` and `process-in-1` respectively. The default output binding is `process-out-0`. +In this example, the first parameter of `BiFunction` is bound as a `KStream` for the first input and the second parameter is bound as a `KTable` for the second input. + +====== BiConsumer in Kafka Streams Binder + +If there are two inputs, but no outputs, in that case we can use `java.util.function.BiConsumer` as shown below. + +[source] +---- +@Bean +public BiConsumer, KTable> process() { + return (userClicksStream, userRegionsTable) -> {} +} +---- + +====== Beyond two inputs + +What if you have more than two inputs? +There are situations in which you need more than two inputs. In that case, the binder allows you to chain partial functions. +In functional programming jargon, this technique is generally known as currying. +With the functional programming support added as part of Java 8, Java now enables you to write curried functions. +Spring Cloud Stream Kafka Streams binder can make use of this feature to enable multiple input bindings. + +Let's see an example. + +[source] +---- +@Bean +public Function, + Function, + Function, KStream>>> enrichOrder() { + + return orders -> ( + customers -> ( + products -> ( + orders.join(customers, + (orderId, order) -> order.getCustomerId(), + (order, customer) -> new CustomerOrder(customer, order)) + .join(products, + (orderId, customerOrder) -> customerOrder + .productId(), + (customerOrder, product) -> { + EnrichedOrder enrichedOrder = new EnrichedOrder(); + enrichedOrder.setProduct(product); + enrichedOrder.setCustomer(customerOrder.customer); + enrichedOrder.setOrder(customerOrder.order); + return enrichedOrder; + }) + ) + ) + ); +} +---- + +Let's look at the details of the binding model presented above. +In this model, we have 3 partially applied functions on the inbound. Let's call them as `f(x)`, `f(y)` and `f(z)`. +If we expand these functions in the sense of true mathematical functions, it will look like these: `f(x) -> (fy) -> f(z) -> KStream`. +The `x` variable stands for `KStream`, the `y` variable stands for `GlobalKTable` and the `z` variable stands for `GlobalKTable`. +The first function `f(x)` has the first input binding of the application (`KStream`) and its output is the function, f(y). +The function `f(y)` has the second input binding for the application (`GlobalKTable`) and its output is yet another function, `f(z)`. +The input for the function `f(z)` is the third input for the application (`GlobalKTable`) and its output is `KStream` which is the final output binding for the application. +The input from the three partial functions which are `KStream`, `GlobalKTable`, `GlobalKTable` respectively are available for you in the method body for implementing the business logic as part of the lambda expression. + +Input bindings are named as `enrichOrder-in-0`, `enrichOrder-in-1` and `enrichOrder-in-2` respectively. Output binding is named as `enrichOrder-out-0`. + +With curried functions, you can virtually have any number of inputs. However, keep in mind that, anything more than a smaller number of inputs and partially applied functions for them as above in Java might lead to unreadable code. +Therefore if your Kafka Streams application requires more than a reasonably smaller number of input bindings, and you want to use this functional model, then you may want to rethink your design and decompose the application appropriately. + +===== Output Bindings + +Kafka Streams binder allows types of either `KStream` or `KTable` as output bindings. +Behind the scenes, the binder uses the `to` method on `KStream` to send the resultant records to the output topic. +If the application provides a `KTable` as output in the function, the binder still uses this technique by delegating to the `to` method of `KStream`. + +For example both functions below will work: + +``` +@Bean +public Function, KTable> foo() { + return KStream::toTable; + }; +} + +@Bean +public Function, KStream> bar() { + return KTable::toStream; +} +``` + +===== Multiple Output Bindings + +Kafka Streams allows writing outbound data into multiple topics. This feature is known as branching in Kafka Streams. +When using multiple output bindings, you need to provide an array of KStream (`KStream[]`) as the outbound return type. + +Here is an example: + +[source] +---- +@Bean +public Function, KStream[]> process() { + + Predicate isEnglish = (k, v) -> v.word.equals("english"); + Predicate isFrench = (k, v) -> v.word.equals("french"); + Predicate isSpanish = (k, v) -> v.word.equals("spanish"); + + return input -> { + final Map> stringKStreamMap = input + .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .groupBy((key, value) -> value) + .windowedBy(TimeWindows.of(Duration.ofSeconds(5))) + .count(Materialized.as("WordCounts-branch")) + .toStream() + .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, + new Date(key.window().start()), new Date(key.window().end())))) + .split() + .branch(isEnglish) + .branch(isFrench) + .branch(isSpanish) + .noDefaultBranch(); + + return stringKStreamMap.values().toArray(new KStream[0]); + }; +} +---- + +The programming model remains the same, however the outbound parameterized type is `KStream[]`. +The default output binding names are `process-out-0`, `process-out-1`, `process-out-2` respectively for the function above. +The reason why the binder generates three output bindings is because it detects the length of the returned `KStream` array as three. +Note that in this example, we provide a `noDefaultBranch()`; if we have used `defaultBranch()` instead, that would have required an extra output binding, essentially returning a `KStream` array of length four. + +===== Summary of Function based Programming Styles for Kafka Streams + +In summary, the following table shows the various options that can be used in the functional paradigm. + +|=== +|Number of Inputs |Number of Outputs |Component to use + +|1|0|java.util.function.Consumer +|2|0|java.util.function.BiConsumer +|1|1..n |java.util.function.Function +|2|1..n |java.util.function.BiFunction +|>= 3 |0..n |Use curried functions + +|=== + +* In the case of more than one output in this table, the type simply becomes `KStream[]`. + +===== Function composition in Kafka Streams binder + +Kafka Streams binder supports minimal forms of functional composition for linear topologies. +Using the Java functional API support, you can write multiple functions and then compose them on your own using the `andThen` method. +For example, assume that you have the following two functions. + +``` +@Bean +public Function, KStream> foo() { + return input -> input.peek((s, s2) -> {}); +} + +@Bean +public Function, KStream> bar() { + return input -> input.peek((s, s2) -> {}); +} +``` + +Even without the functional composition support in the binder, you can compose these two functions as below. + +``` +@Bean +pubic Funcion, KStream> composed() { + foo().andThen(bar()); +} +``` + +Then you can provide deefinitions of the form `spring.cloud.stream.function.definition=foo;bar;composed`. +With the functional composition support in the binder, you don't need to write this third function in which you are doing explicit function composition. + +You can simply do this instead: + +``` +spring.cloud.stream.function.definition=foo|bar +``` + +You can even do this: + +``` +spring.cloud.stream.function.definition=foo|bar;foo;bar +``` + +The composed function's default binding names in this example becomes `foobar-in-0` and `foobar-out-0`. + +====== Limitations of functional composition in Kafka Streams bincer + +When you have `java.util.function.Function` bean, that can be composed with another function or multiple functions. +The same function bean can be composed with a `java.util.function.Consumer` as well. In this case, consumer is the last component composed. +A function can be composed with multiple functions, then end with a `java.util.function.Consumer` bean as well. + +When composing the beans of type `java.util.function.BiFunction`, the `BiFunction` must be the first function in the definition. +The composed entities must be either of type `java.util.function.Function` or `java.util.funciton.Consumer`. +In other words, you cannot take a `BiFunction` bean and then compose with another `BiFunction`. + +You cannot compose with types of `BiConsumer` or definitions where `Consumer` is the first component. +You cannot also compose with functions where the output is an array (`KStream[]` for branching) unless this is the last component in the definition. + +The very first `Function` of `BiFunction` in the function definition may use a curried form also. +For example, the following is possible. + +``` +@Bean +public Function, Function, KStream>> curriedFoo() { + return a -> b -> + a.join(b, (value1, value2) -> value1 + value2); +} + +@Bean +public Function, KStream> bar() { + return input -> input.mapValues(value -> value + "From-anotherFooFunc"); +} +``` + +and the function definition could be `curriedFoo|bar`. +Behind the scenes, the binder will create two input bindings for the curried function, and an output binding based on the final function in the definition. +The default input bindings in this case are going to be `curriedFoobar-in-0` and `curriedFoobar-in-1`. +The default output binding for this example becomes `curriedFoobar-out-0`. + +====== Special note on using `KTable` as output in function composition + +Lets say you have the following two functions. + +``` +@Bean +public Function, KTable> foo() { + return KStream::toTable; + }; +} + +@Bean +public Function, KStream> bar() { + return KTable::toStream; +} +``` + +You can compose them as `foo|bar`, but keep in mind that the second function (`bar` in this case) must have a `KTable` as input since the first function (`foo`) has `KTable` as output. + +==== Imperative programming model. + +Starting with `3.1.0` version of the binder, we recommend using the functional programming model described above for Kafka Streams binder based applications. +The support for `StreamListener` is deprecated starting with `3.1.0` of Spring Cloud Stream. +Below, we are providing some details on the `StreamListener` based Kafka Streams processors as a reference. + +Following is the equivalent of the Word count example using `StreamListener`. + +[source] +---- +@SpringBootApplication +@EnableBinding(KafkaStreamsProcessor.class) +public class WordCountProcessorApplication { + + @StreamListener("input") + @SendTo("output") + public KStream process(KStream input) { + return input + .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .groupBy((key, value) -> value) + .windowedBy(TimeWindows.of(5000)) + .count(Materialized.as("WordCounts-multi")) + .toStream() + .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))); + } + + public static void main(String[] args) { + SpringApplication.run(WordCountProcessorApplication.class, args); + } +---- + +As you can see, this is a bit more verbose since you need to provide `EnableBinding` and the other extra annotations like `StreamListener` and `SendTo` to make it a complete application. +`EnableBinding` is where you specify your binding interface that contains your bindings. +In this case, we are using the stock `KafkaStreamsProcessor` binding interface that has the following contracts. + +[source] +---- +public interface KafkaStreamsProcessor { + + @Input("input") + KStream input(); + + @Output("output") + KStream output(); + +} +---- + +Binder will create bindings for the input `KStream` and output `KStream` since you are using a binding interface that contains those declarations. + +In addition to the obvious differences in the programming model offered in the functional style, one particular thing that needs to be mentioned here is that the binding names are what you specify in the binding interface. +For example, in the above application, since we are using `KafkaStreamsProcessor`, the binding names are `input` and `output`. +Binding properties need to use those names. For instance `spring.cloud.stream.bindings.input.destination`, `spring.cloud.stream.bindings.output.destination` etc. +Keep in mind that this is fundamentally different from the functional style since there the binder generates binding names for the application. +This is because the application does not provide any binding interfaces in the functional model using `EnableBinding`. + +Here is another example of a sink where we have two inputs. + +[source] +---- +@EnableBinding(KStreamKTableBinding.class) +..... +..... +@StreamListener +public void process(@Input("inputStream") KStream playEvents, + @Input("inputTable") KTable songTable) { + .... + .... +} + +interface KStreamKTableBinding { + + @Input("inputStream") + KStream inputStream(); + + @Input("inputTable") + KTable inputTable(); +} + +---- + +Following is the `StreamListener` equivalent of the same `BiFunction` based processor that we saw above. + + +[source] +---- +@EnableBinding(KStreamKTableBinding.class) +.... +.... + +@StreamListener +@SendTo("output") +public KStream process(@Input("input") KStream userClicksStream, + @Input("inputTable") KTable userRegionsTable) { +.... +.... +} + +interface KStreamKTableBinding extends KafkaStreamsProcessor { + + @Input("inputX") + KTable inputTable(); +} +---- + +Finally, here is the `StreamListener` equivalent of the application with three inputs and curried functions. + +[source] +---- +@EnableBinding(CustomGlobalKTableProcessor.class) +... +... + @StreamListener + @SendTo("output") + public KStream process( + @Input("input-1") KStream ordersStream, + @Input("input-2") GlobalKTable customers, + @Input("input-3") GlobalKTable products) { + + KStream customerOrdersStream = ordersStream.join( + customers, (orderId, order) -> order.getCustomerId(), + (order, customer) -> new CustomerOrder(customer, order)); + + return customerOrdersStream.join(products, + (orderId, customerOrder) -> customerOrder.productId(), + (customerOrder, product) -> { + EnrichedOrder enrichedOrder = new EnrichedOrder(); + enrichedOrder.setProduct(product); + enrichedOrder.setCustomer(customerOrder.customer); + enrichedOrder.setOrder(customerOrder.order); + return enrichedOrder; + }); + } + + interface CustomGlobalKTableProcessor { + + @Input("input-1") + KStream input1(); + + @Input("input-2") + GlobalKTable input2(); + + @Input("input-3") + GlobalKTable input3(); + + @Output("output") + KStream output(); + } + +---- + +You might notice that the above two examples are even more verbose since in addition to provide `EnableBinding`, you also need to write your own custom binding interface as well. +Using the functional model, you can avoid all those ceremonial details. + +Before we move on from looking at the general programming model offered by Kafka Streams binder, here is the `StreamListener` version of multiple output bindings. + +[source] +---- +EnableBinding(KStreamProcessorWithBranches.class) +public static class WordCountProcessorApplication { + + @Autowired + private TimeWindows timeWindows; + + @StreamListener("input") + @SendTo({"output1","output2","output3"}) + public KStream[] process(KStream input) { + + Predicate isEnglish = (k, v) -> v.word.equals("english"); + Predicate isFrench = (k, v) -> v.word.equals("french"); + Predicate isSpanish = (k, v) -> v.word.equals("spanish"); + + return input + .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .groupBy((key, value) -> value) + .windowedBy(timeWindows) + .count(Materialized.as("WordCounts-1")) + .toStream() + .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))) + .branch(isEnglish, isFrench, isSpanish); + } + + interface KStreamProcessorWithBranches { + + @Input("input") + KStream input(); + + @Output("output1") + KStream output1(); + + @Output("output2") + KStream output2(); + + @Output("output3") + KStream output3(); + } +} +---- + +To recap, we have reviewed the various programming model choices when using the Kafka Streams binder. + +The binder provides binding capabilities for `KStream`, `KTable` and `GlobalKTable` on the input. +`KTable` and `GlobalKTable` bindings are only available on the input. +Binder supports both input and output bindings for `KStream`. + +The upshot of the programming model of Kafka Streams binder is that the binder provides you the flexibility of going with a fully functional programming model or using the `StreamListener` based imperative approach. + +=== Ancillaries to the programming model + +==== Multiple Kafka Streams processors within a single application + +Binder allows to have multiple Kafka Streams processors within a single Spring Cloud Stream application. +You can have an application as below. + +``` +@Bean +public java.util.function.Function, KStream> process() { + ... +} + +@Bean +public java.util.function.Consumer> anotherProcess() { + ... +} + +@Bean +public java.util.function.BiFunction, KTable, KStream> yetAnotherProcess() { + ... +} + +``` + +In this case, the binder will create 3 separate Kafka Streams objects with different application ID's (more on this below). +However, if you have more than one processor in the application, you have to tell Spring Cloud Stream, which functions need to be activated. +Here is how you activate the functions. + +`spring.cloud.stream.function.definition: process;anotherProcess;yetAnotherProcess` + +If you want certain functions to be not activated right away, you can remove that from this list. + +This is also true when you have a single Kafka Streams processor and other types of `Function` beans in the same application that is handled through a different binder (for e.g., a function bean that is based on the regular Kafka Message Channel binder) + +==== Kafka Streams Application ID + +Application id is a mandatory property that you need to provide for a Kafka Streams application. +Spring Cloud Stream Kafka Streams binder allows you to configure this application id in multiple ways. + +If you only have one single processor or `StreamListener` in the application, then you can set this at the binder level using the following property: + +`spring.cloud.stream.kafka.streams.binder.applicationId`. + +As a convenience, if you only have a single processor, you can also use `spring.application.name` as the property to delegate the application id. + +If you have multiple Kafka Streams processors in the application, then you need to set the application id per processor. +In the case of the functional model, you can attach it to each function as a property. + +For e.g. imagine that you have the following functions. + +``` +@Bean +public java.util.function.Consumer> process() { + ... +} +``` + +and + +``` +@Bean +public java.util.function.Consumer> anotherProcess() { + ... +} +``` + +Then you can set the application id for each, using the following binder level properties. + +`spring.cloud.stream.kafka.streams.binder.functions.process.applicationId` + +and + +`spring.cloud.stream.kafka.streams.binder.functions.anotherProcess.applicationId` + +In the case of `StreamListener`, you need to set this on the first input binding on the processor. + +For e.g. imagine that you have the following two `StreamListener` based processors. + +``` +@StreamListener +@SendTo("output") +public KStream process(@Input("input") > input) { + ... +} + +@StreamListener +@SendTo("anotherOutput") +public KStream anotherProcess(@Input("anotherInput") > input) { + ... +} +``` + +Then you must set the application id for this using the following binding property. + +`spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId` + +and + +`spring.cloud.stream.kafka.streams.bindings.anotherInput.consumer.applicationId` + + +For function based model also, this approach of setting application id at the binding level will work. +However, setting per function at the binder level as we have seen above is much easier if you are using the functional model. + +For production deployments, it is highly recommended to explicitly specify the application ID through configuration. +This is especially going to be very critical if you are auto scaling your application in which case you need to make sure that you are deploying each instance with the same application ID. + +If the application does not provide an application ID, then in that case the binder will auto generate a static application ID for you. +This is convenient in development scenarios as it avoids the need for explicitly providing the application ID. +The generated application ID in this manner will be static over application restarts. +In the case of functional model, the generated application ID will be the function bean name followed by the literal `applicationID`, for e.g `process-applicationID` if `process` if the function bean name. +In the case of `StreamListener`, instead of using the function bean name, the generated application ID will be use the containing class name followed by the method name followed by the literal `applicationId`. + +====== Summary of setting Application ID + +* By default, binder will auto generate the application ID per function or `StreamListener` methods. +* If you have a single processor, then you can use `spring.kafka.streams.applicationId`, `spring.application.name` or `spring.cloud.stream.kafka.streams.binder.applicationId`. +* If you have multiple processors, then application ID can be set per function using the property - `spring.cloud.stream.kafka.streams.binder.functions..applicationId`. +In the case of `StreamListener`, this can be done using `spring.cloud.stream.kafka.streams.bindings.input.applicationId`, assuming that the input binding name is `input`. + +==== Overriding the default binding names generated by the binder with the functional style + +By default, the binder uses the strategy discussed above to generate the binding name when using the functional style, i.e. -|-[0..n], for e.g. process-in-0, process-out-0 etc. +If you want to override those binding names, you can do that by specifying the following properties. + +`spring.cloud.stream.function.bindings.`. Default binding name is the original binding name generated by the binder. + +For e.g. lets say, you have this function. + +[source] +---- +@Bean +public BiFunction, KTable, KStream> process() { +... +} +---- + +Binder will generate bindings with names, `process-in-0`, `process-in-1` and `process-out-0`. +Now, if you want to change them to something else completely, maybe more domain specific binding names, then you can do so as below. + +`spring.cloud.stream.function.bindings.process-in-0=users` + +`spring.cloud.stream.function.bindings.process-in-0=regions` + +and + +`spring.cloud.stream.function.bindings.process-out-0=clicks` + +After that, you must set all the binding level properties on these new binding names. + +Please keep in mind that with the functional programming model described above, adhering to the default binding names make sense in most situations. +The only reason you may still want to do this overriding is when you have larger number of configuration properties and you want to map the bindings to something more domain friendly. + +==== Setting up bootstrap server configuration + +When running Kafka Streams applications, you must provide the Kafka broker server information. +If you don't provide this information, the binder expects that you are running the broker at the default `localhost:9092`. +If that is not the case, then you need to override that. There are a couple of ways to do that. + +* Using the boot property - `spring.kafka.bootstrapServers` +* Binder level property - `spring.cloud.stream.kafka.streams.binder.brokers` + +When it comes to the binder level property, it doesn't matter if you use the broker property provided through the regular Kafka binder - `spring.cloud.stream.kafka.binder.brokers`. +Kafka Streams binder will first check if Kafka Streams binder specific broker property is set (`spring.cloud.stream.kafka.streams.binder.brokers`) and if not found, it looks for `spring.cloud.stream.kafka.binder.brokers`. + +=== Record serialization and deserialization + +Kafka Streams binder allows you to serialize and deserialize records in two ways. +One is the native serialization and deserialization facilities provided by Kafka and the other one is the message conversion capabilities of Spring Cloud Stream framework. +Lets look at some details. + +==== Inbound deserialization + +Keys are always deserialized using native Serdes. + +For values, by default, deserialization on the inbound is natively performed by Kafka. +Please note that this is a major change on default behavior from previous versions of Kafka Streams binder where the deserialization was done by the framework. + +Kafka Streams binder will try to infer matching `Serde` types by looking at the type signature of `java.util.function.Function|Consumer` or `StreamListener`. +Here is the order that it matches the Serdes. + +* If the application provides a bean of type `Serde` and if the return type is parameterized with the actual type of the incoming key or value type, then it will use that `Serde` for inbound deserialization. +For e.g. if you have the following in the application, the binder detects that the incoming value type for the `KStream` matches with a type that is parameterized on a `Serde` bean. +It will use that for inbound deserialization. + + +``` +@Bean +public Serde customSerde() { + ... +} + +@Bean +public Function, KStream> process() { +} +``` + +* Next, it looks at the types and see if they are one of the types exposed by Kafka Streams. If so, use them. +Here are the Serde types that the binder will try to match from Kafka Streams. + + Integer, Long, Short, Double, Float, byte[], UUID and String. + +* If none of the Serdes provided by Kafka Streams don't match the types, then it will use JsonSerde provided by Spring Kafka. In this case, the binder assumes that the types are JSON friendly. +This is useful if you have multiple value objects as inputs since the binder will internally infer them to correct Java types. +Before falling back to the `JsonSerde` though, the binder checks at the default `Serde`s set in the Kafka Streams configuration to see if it is a `Serde` that it can match with the incoming KStream's types. + +If none of the above strategies worked, then the applications must provide the `Serde`s through configuration. +This can be configured in two ways - binding or default. + +First the binder will look if a `Serde` is provided at the binding level. +For e.g. if you have the following processor, + +``` +@Bean +public BiFunction, KTable, KStream> process() {...} +``` + +then, you can provide a binding level `Serde` using the following: + +``` +spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.keySerde=CustomKeySerde +spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.valueSerde=io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde + +spring.cloud.stream.kafka.streams.bindings.process-in-1.consumer.keySerde=CustomKeySerde +spring.cloud.stream.kafka.streams.bindings.process-in-1.consumer.valueSerde=io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde +``` + +NOTE: If you provide `Serde` as abover per input binding, then that will takes higher precedence and the binder will stay away from any `Serde` inference. + +If you want the default key/value Serdes to be used for inbound deserialization, you can do so at the binder level. + +``` +spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde +spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde +``` + +If you don't want the native decoding provided by Kafka, you can rely on the message conversion features that Spring Cloud Stream provides. +Since native decoding is the default, in order to let Spring Cloud Stream deserialize the inbound value object, you need to explicitly disable native decoding. + +For e.g. if you have the same BiFunction processor as above, then `spring.cloud.stream.bindings.process-in-0.consumer.nativeDecoding: false` +You need to disable native decoding for all the inputs individually. Otherwise, native decoding will still be applied for those you do not disable. + +By default, Spring Cloud Stream will use `application/json` as the content type and use an appropriate json message converter. +You can use custom message converters by using the following property and an appropriate `MessageConverter` bean. +``` +spring.cloud.stream.bindings.process-in-0.contentType +``` + +==== Outbound serialization + +Outbound serialization pretty much follows the same rules as above for inbound deserialization. +As with the inbound deserialization, one major change from the previous versions of Spring Cloud Stream is that the serialization on the outbound is handled by Kafka natively. +Before 3.0 versions of the binder, this was done by the framework itself. + +Keys on the outbound are always serialized by Kafka using a matching `Serde` that is inferred by the binder. +If it can't infer the type of the key, then that needs to be specified using configuration. + +Value serdes are inferred using the same rules used for inbound deserialization. +First it matches to see if the outbound type is from a provided bean in the application. +If not, it checks to see if it matches with a `Serde` exposed by Kafka such as - `Integer`, `Long`, `Short`, `Double`, `Float`, `byte[]`, `UUID` and `String`. +If that doesnt't work, then it falls back to `JsonSerde` provided by the Spring Kafka project, but first look at the default `Serde` configuration to see if there is a match. +Keep in mind that all these happen transparently to the application. +If none of these work, then the user has to provide the `Serde` to use by configuration. + +Lets say you are using the same `BiFunction` processor as above. Then you can configure outbound key/value Serdes as following. + +``` +spring.cloud.stream.kafka.streams.bindings.process-out-0.producer.keySerde=CustomKeySerde +spring.cloud.stream.kafka.streams.bindings.process-out-0.producer.valueSerde=io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde +``` + +If Serde inference fails, and no binding level Serdes are provided, then the binder falls back to the `JsonSerde`, but look at the default Serdes for a match. + +Default serdes are configured in the same way as above where it is described under deserialization. + +`spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde` +`spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde` + +If your application uses the branching feature and has multiple output bindings, then these have to be configured per binding. +Once again, if the binder is capable of inferring the `Serde` types, you don't need to do this configuration. + +If you don't want the native encoding provided by Kafka, but want to use the framework provided message conversion, then you need to explicitly disable native encoding since since native encoding is the default. +For e.g. if you have the same BiFunction processor as above, then `spring.cloud.stream.bindings.process-out-0.producer.nativeEncoding: false` +You need to disable native encoding for all the output individually in the case of branching. Otherwise, native encoding will still be applied for those you don't disable. + +When conversion is done by Spring Cloud Stream, by default, it will use `application/json` as the content type and use an appropriate json message converter. +You can use custom message converters by using the following property and a corresponding `MessageConverter` bean. +``` +spring.cloud.stream.bindings.process-out-0.contentType +``` + +When native encoding/decoding is disabled, binder will not do any inference as in the case of native Serdes. +Applications need to explicitly provide all the configuration options. +For that reason, it is generally advised to stay with the default options for de/serialization and stick with native de/serialization provided by Kafka Streams when you write Spring Cloud Stream Kafka Streams applications. +The one scenario in which you must use message conversion capabilities provided by the framework is when your upstream producer is using a specific serialization strategy. +In that case, you want to use a matching deserialization strategy as native mechanisms may fail. +When relying on the default `Serde` mechanism, the applications must ensure that the binder has a way forward with correctly map the inbound and outbound with a proper `Serde`, as otherwise things might fail. + +It is worth to mention that the data de/serialization approaches outlined above are only applicable on the edges of your processors, i.e. - inbound and outbound. +Your business logic might still need to call Kafka Streams API's that explicitly need `Serde` objects. +Those are still the responsibility of the application and must be handled accordingly by the developer. + +=== Error Handling + +Apache Kafka Streams provides the capability for natively handling exceptions from deserialization errors. +For details on this support, please see https://cwiki.apache.org/confluence/display/KAFKA/KIP-161%3A+streams+deserialization+exception+handlers[this]. +Out of the box, Apache Kafka Streams provides two kinds of deserialization exception handlers - `LogAndContinueExceptionHandler` and `LogAndFailExceptionHandler`. +As the name indicates, the former will log the error and continue processing the next records and the latter will log the error and fail. `LogAndFailExceptionHandler` is the default deserialization exception handler. + +==== Handling Deserialization Exceptions in the Binder + +Kafka Streams binder allows to specify the deserialization exception handlers above using the following property. + +[source] +---- +spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler: logAndContinue +---- + +or + +[source] +---- +spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler: logAndFail +---- + +In addition to the above two deserialization exception handlers, the binder also provides a third one for sending the erroneous records (poison pills) to a DLQ (dead letter queue) topic. +Here is how you enable this DLQ exception handler. + +[source] +---- +spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler: sendToDlq +---- + +When the above property is set, all the records in deserialization error are automatically sent to the DLQ topic. + +You can set the topic name where the DLQ messages are published as below. + +You can provide an implementation for `DlqDestinationResolver` which is a functional interface. +`DlqDestinationResolver` takes `ConsumerRecord` and the exception as inputs and then allows to specify a topic name as the output. +By gaining access to the Kafka `ConsumerRecord`, the header records can be introspected in the implementation of the `BiFunction`. + +Here is an example of providing an implementation for `DlqDestinationResolver`. + +[source] +---- +@Bean +public DlqDestinationResolver dlqDestinationResolver() { + return (rec, ex) -> { + if (rec.topic().equals("word1")) { + return "topic1-dlq"; + } + else { + return "topic2-dlq"; + } + }; +} +---- + +One important thing to keep in mind when providing an implementation for `DlqDestinationResolver` is that the provisioner in the binder will not auto create topics for the application. +This is because there is no way for the binder to infer the names of all the DLQ topics the implementation might send to. +Therefore, if you provide DLQ names using this strategy, it is the application's responsibility to ensure that those topics are created beforehand. + +If `DlqDestinationResolver` is present in the application as a bean, that takes higher precedence. +If you do not want to follow this approach and rather provide a static DLQ name using configuration, you can set the following property. + +[source] +---- +spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.dlqName: custom-dlq (Change the binding name accordingly) +---- + +If this is set, then the error records are sent to the topic `custom-dlq`. +If the application is not using either of the above strategies, then it will create a DLQ topic with the name `error..`. +For instance, if your binding's destination topic is `inputTopic` and the application ID is `process-applicationId`, then the default DLQ topic is `error.inputTopic.process-applicationId`. +It is always recommended to explicitly create a DLQ topic for each input binding if it is your intention to enable DLQ. + +==== DLQ per input consumer binding + +The property `spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler` is applicable for the entire application. +This implies that if there are multiple functions or `StreamListener` methods in the same application, this property is applied to all of them. +However, if you have multiple processors or multiple input bindings within a single processor, then you can use the finer-grained DLQ control that the binder provides per input consumer binding. + +If you have the following processor, + +``` +@Bean +public BiFunction, KTable, KStream> process() { +... +} +``` + +and you only want to enable DLQ on the first input binding and skipAndContinue on the second binding, then you can do so on the consumer as below. + +`spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.deserializationExceptionHandler: sendToDlq` +`spring.cloud.stream.kafka.streams.bindings.process-in-1.consumer.deserializationExceptionHandler: skipAndContinue` + +Setting deserialization exception handlers this way has a higher precedence than setting at the binder level. + +==== DLQ partitioning + +By default, records are published to the Dead-Letter topic using the same partition as the original record. +This means the Dead-Letter topic must have at least as many partitions as the original record. + +To change this behavior, add a `DlqPartitionFunction` implementation as a `@Bean` to the application context. +Only one such bean can be present. +The function is provided with the consumer group (which is the same as the application ID in most situations), the failed `ConsumerRecord` and the exception. +For example, if you always want to route to partition 0, you might use: + + +[source, java] +---- +@Bean +public DlqPartitionFunction partitionFunction() { + return (group, record, ex) -> 0; +} +---- + +NOTE: If you set a consumer binding's `dlqPartitions` property to 1 (and the binder's `minPartitionCount` is equal to `1`), there is no need to supply a `DlqPartitionFunction`; the framework will always use partition 0. +If you set a consumer binding's `dlqPartitions` property to a value greater than `1` (or the binder's `minPartitionCount` is greater than `1`), you **must** provide a `DlqPartitionFunction` bean, even if the partition count is the same as the original topic's. + + +A couple of things to keep in mind when using the exception handling feature in Kafka Streams binder. + +* The property `spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler` is applicable for the entire application. +This implies that if there are multiple functions or `StreamListener` methods in the same application, this property is applied to all of them. +* The exception handling for deserialization works consistently with native deserialization and framework provided message conversion. + +==== Handling Production Exceptions in the Binder + +Unlike the support for deserialization exception handlers as described above, the binder does not provide such first class mechanisms for handling production exceptions. +However, you still can configure production exception handlers using the `StreamsBuilderFactoryBean` customizer which you can find more details about, in a subsequent section below. + +=== Retrying critical business logic + +There are scenarios in which you might want to retry parts of your business logic that are critical to the application. +There maybe an external call to a relational database or invoking a REST endpoint from the Kafka Streams processor. +These calls can fail for various reasons such as network issues or remote service unavailability. +More often, these failures may self resolve if you can try them again. +By default, Kafka Streams binder creates `RetryTemplate` beans for all the input bindings. + +If the function has the following signature, +``` +@Bean +public java.util.function.Consumer> process() +``` +and with default binding name, the `RetryTemplate` will be registered as `process-in-0-RetryTemplate`. +This is following the convention of binding name (`process-in-0`) followed by the literal `-RetryTemplate`. +In the case of multiple input bindings, there will be a separate `RetryTemplate` bean available per binding. +If there is a custom `RetryTemplate` bean available in the application and provided through `spring.cloud.stream.bindings..consumer.retryTemplateName`, then that takes precedence over any input binding level retry template configuration properties. + +Once the `RetryTemplate` from the binding is injected into the application, it can be used to retry any critical sections of the application. +Here is an example: + +``` +@Bean +public java.util.function.Consumer> process(@Lazy @Qualifier("process-in-0-RetryTemplate") RetryTemplate retryTemplate) { + + return input -> input + .process(() -> new Processor() { + @Override + public void init(ProcessorContext processorContext) { + } + + @Override + public void process(Object o, String s) { + retryTemplate.execute(context -> { + //Critical business logic goes here. + }); + } + + @Override + public void close() { + } + }); +} +``` + +Or you can use a custom `RetryTemplate` as below. + +``` +@EnableAutoConfiguration +public static class CustomRetryTemplateApp { + + @Bean + @StreamRetryTemplate + RetryTemplate fooRetryTemplate() { + RetryTemplate retryTemplate = new RetryTemplate(); + + RetryPolicy retryPolicy = new SimpleRetryPolicy(4); + FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy(); + backOffPolicy.setBackOffPeriod(1); + + retryTemplate.setBackOffPolicy(backOffPolicy); + retryTemplate.setRetryPolicy(retryPolicy); + + return retryTemplate; + } + + @Bean + public java.util.function.Consumer> process() { + + return input -> input + .process(() -> new Processor() { + @Override + public void init(ProcessorContext processorContext) { + } + + @Override + public void process(Object o, String s) { + fooRetryTemplate().execute(context -> { + //Critical business logic goes here. + }); + + } + + @Override + public void close() { + } + }); + } +} +``` + +Note that when retries are exhausted, by default, the last exception will be thrown, causing the processor to terminate. +If you wish to handle the exception and continue processing, you can add a RecoveryCallback to the `execute` method: +Here is an example. +``` +retryTemplate.execute(context -> { + //Critical business logic goes here. + }, context -> { + //Recovery logic goes here. + return null; + )); +``` +Refer to the https://github.com/spring-projects/spring-retry[Spring Retry] project for more information about the RetryTemplate, retry policies, backoff policies and more. + +=== State Store + +State stores are created automatically by Kafka Streams when the high level DSL is used and appropriate calls are made those trigger a state store. + +If you want to materialize an incoming `KTable` binding as a named state store, then you can do so by using the following strategy. + +Lets say you have the following function. + +[source] +---- +@Bean +public BiFunction, KTable, KStream> process() { + ... +} +---- + +Then by setting the following property, the incoming `KTable` data will be materialized in to the named state store. + +[source] +---- +spring.cloud.stream.kafka.streams.bindings.process-in-1.consumer.materializedAs: incoming-store +---- + +You can define custom state stores as beans in your application and those will be detected and added to the Kafka Streams builder by the binder. +Especially when the processor API is used, you need to register a state store manually. +In order to do so, you can create the StateStore as a bean in the application. +Here are examples of defining such beans. + +[source] +---- +@Bean +public StoreBuilder myStore() { + return Stores.keyValueStoreBuilder( + Stores.persistentKeyValueStore("my-store"), Serdes.Long(), + Serdes.Long()); +} + +@Bean +public StoreBuilder otherStore() { + return Stores.windowStoreBuilder( + Stores.persistentWindowStore("other-store", + 1L, 3, 3L, false), Serdes.Long(), + Serdes.Long()); +} +---- + +These state stores can be then accessed by the applications directly. + +During the bootstrap, the above beans will be processed by the binder and passed on to the Streams builder object. + +Accessing the state store: +[source] +---- +Processor() { + + WindowStore state; + + @Override + public void init(ProcessorContext processorContext) { + state = (WindowStore)processorContext.getStateStore("mystate"); + } + ... +} +---- + +This will not work when it comes to registering global state stores. +In order to register a global state store, please see the section below on customizing `StreamsBuilderFactoryBean`. + +=== Interactive Queries + +Kafka Streams binder API exposes a class called `InteractiveQueryService` to interactively query the state stores. +You can access this as a Spring bean in your application. An easy way to get access to this bean from your application is to `autowire` the bean. + +[source] +---- +@Autowired +private InteractiveQueryService interactiveQueryService; +---- + +Once you gain access to this bean, then you can query for the particular state-store that you are interested. See below. + +[source] +---- +ReadOnlyKeyValueStore keyValueStore = + interactiveQueryService.getQueryableStoreType("my-store", QueryableStoreTypes.keyValueStore()); +---- + +During the startup, the above method call to retrieve the store might fail. +For example, it might still be in the middle of initializing the state store. +In such cases, it will be useful to retry this operation. +Kafka Streams binder provides a simple retry mechanism to accommodate this. + +Following are the two properties that you can use to control this retrying. + +* spring.cloud.stream.kafka.streams.binder.stateStoreRetry.maxAttempts - Default is `1` . +* spring.cloud.stream.kafka.streams.binder.stateStoreRetry.backOffInterval - Default is `1000` milliseconds. + +If there are multiple instances of the kafka streams application running, then before you can query them interactively, you need to identify which application instance hosts the particular key that you are querying. +`InteractiveQueryService` API provides methods for identifying the host information. + +In order for this to work, you must configure the property `application.server` as below: + +[source] +---- +spring.cloud.stream.kafka.streams.binder.configuration.application.server: : +---- + +Here are some code snippets: + +[source] +---- +org.apache.kafka.streams.state.HostInfo hostInfo = interactiveQueryService.getHostInfo("store-name", + key, keySerializer); + +if (interactiveQueryService.getCurrentHostInfo().equals(hostInfo)) { + + //query from the store that is locally available +} +else { + //query from the remote host +} +---- + +For more information on these host finding methods, please see the Javadoc on the methods. +For these methods also, during startup, if the underlying KafkaStreams objects are not ready, they might throw exceptions. +The aforementioned retry properties are applicable for these methods as well. + +==== Other API methods available through the InteractiveQueryService + +Use the following API method to retrieve the `KeyQueryMetadata` object associated with the combination of given store and key. + +``` +public KeyQueryMetadata getKeyQueryMetadata(String store, K key, Serializer serializer) +``` + +Use the following API method to retrieve the `KakfaStreams` object associated with the combination of given store and key. + +``` +public KafkaStreams getKafkaStreams(String store, K key, Serializer serializer) +``` + +=== Health Indicator + +The health indicator requires the dependency `spring-boot-starter-actuator`. For maven use: +[source,xml] +---- + + org.springframework.boot + spring-boot-starter-actuator + +---- + +Spring Cloud Stream Kafka Streams Binder provides a health indicator to check the state of the underlying streams threads. +Spring Cloud Stream defines a property `management.health.binders.enabled` to enable the health indicator. See the +https://docs.spring.io/spring-cloud-stream/docs/current/reference/htmlsingle/#_health_indicator[Spring Cloud Stream documentation]. + +The health indicator provides the following details for each stream thread's metadata: + +* Thread name +* Thread state: `CREATED`, `RUNNING`, `PARTITIONS_REVOKED`, `PARTITIONS_ASSIGNED`, `PENDING_SHUTDOWN` or `DEAD` +* Active tasks: task ID and partitions +* Standby tasks: task ID and partitions + +By default, only the global status is visible (`UP` or `DOWN`). To show the details, the property `management.endpoint.health.show-details` must be set to `ALWAYS` or `WHEN_AUTHORIZED`. +For more details about the health information, see the +https://docs.spring.io/spring-boot/docs/current/reference/html/production-ready-endpoints.html#production-ready-health[Spring Boot Actuator documentation]. + +NOTE: The status of the health indicator is `UP` if all the Kafka threads registered are in the `RUNNING` state. + +Since there are three individual binders in Kafka Streams binder (`KStream`, `KTable` and `GlobalKTable`), all of them will report the health status. +When enabling `show-details`, some of the information reported may be redundant. + +When there are multiple Kafka Streams processors present in the same application, then the health checks will be reported for all of them and will be categorized by the application ID of Kafka Streams. + +=== Accessing Kafka Streams Metrics + +Spring Cloud Stream Kafka Streams binder provides Kafka Streams metrics which can be exported through a Micrometer `MeterRegistry`. + +For Spring Boot version 2.2.x, the metrics support is provided through a custom Micrometer metrics implementation by the binder. +For Spring Boot version 2.3.x, the Kafka Streams metrics support is provided natively through Micrometer. + +When accessing metrics through the Boot actuator endpoint, make sure to add `metrics` to the property `management.endpoints.web.exposure.include`. +Then you can access `/acutator/metrics` to get a list of all the available metrics, which then can be individually accessed through the same URI (`/actuator/metrics/`). + +=== Mixing high level DSL and low level Processor API + +Kafka Streams provides two variants of APIs. +It has a higher level DSL like API where you can chain various operations that maybe familiar to a lot of functional programmers. +Kafka Streams also gives access to a low level Processor API. +The processor API, although very powerful and gives the ability to control things in a much lower level, is imperative in nature. +Kafka Streams binder for Spring Cloud Stream, allows you to use either the high level DSL or mixing both the DSL and the processor API. +Mixing both of these variants give you a lot of options to control various use cases in an application. +Applications can use the `transform` or `process` method API calls to get access to the processor API. + +Here is a look at how one may combine both the DSL and the processor API in a Spring Cloud Stream application using the `process` API. + +``` +@Bean +public Consumer> process() { + return input -> + input.process(() -> new Processor() { + @Override + @SuppressWarnings("unchecked") + public void init(ProcessorContext context) { + this.context = context; + } + + @Override + public void process(Object key, String value) { + //business logic + } + + @Override + public void close() { + + }); +} +``` + +Here is an example using the `transform` API. + +``` +@Bean +public Consumer> process() { + return (input, a) -> + input.transform(() -> new Transformer>() { + @Override + public void init(ProcessorContext context) { + + } + + @Override + public void close() { + + } + + @Override + public KeyValue transform(Object key, String value) { + // business logic - return transformed KStream; + } + }); +} +``` + +The `process` API method call is a terminal operation while the `transform` API is non terminal and gives you a potentially transformed `KStream` using which you can continue further processing using either the DSL or the processor API. + +=== Partition support on the outbound + +A Kafka Streams processor usually sends the processed output into an outbound Kafka topic. +If the outbound topic is partitioned and the processor needs to send the outgoing data into particular partitions, the applications needs to provide a bean of type `StreamPartitioner`. +See https://kafka.apache.org/23/javadoc/org/apache/kafka/streams/processor/StreamPartitioner.html[StreamPartitioner] for more details. +Let's see some examples. + +This is the same processor we already saw multiple times, + +``` +@Bean +public Function, KStream> process() { + + ... +} +``` + +Here is the output binding destination: + +``` +spring.cloud.stream.bindings.process-out-0.destination: outputTopic +``` + +If the topic `outputTopic` has 4 partitions, if you don't provide a partitioning strategy, Kafka Streams will use default partitioning strategy which may not be the outcome you want depending on the particular use case. +Let's say, you want to send any key that matches to `spring` to partition 0, `cloud` to partition 1, `stream` to partition 2, and everything else to partition 3. +This is what you need to do in the application. + +``` +@Bean +public StreamPartitioner streamPartitioner() { + return (t, k, v, n) -> { + if (k.equals("spring")) { + return 0; + } + else if (k.equals("cloud")) { + return 1; + } + else if (k.equals("stream")) { + return 2; + } + else { + return 3; + } + }; +} +``` + +This is a rudimentary implementation, however, you have access to the key/value of the record, the topic name and the total number of partitions. +Therefore, you can implement complex partitioning strategies if need be. + +You also need to provide this bean name along with the application configuration. + +``` +spring.cloud.stream.kafka.streams.bindings.process-out-0.producer.streamPartitionerBeanName: streamPartitioner +``` + +Each output topic in the application needs to be configured separately like this. + +=== StreamsBuilderFactoryBean customizer + +It is often required to customize the `StreamsBuilderFactoryBean` that creates the `KafkaStreams` objects. +Based on the underlying support provided by Spring Kafka, the binder allows you to customize the `StreamsBuilderFactoryBean`. +You can use the `StreamsBuilderFactoryBeanCustomizer` to customize the `StreamsBuilderFactoryBean` itself. +Then, once you get access to the `StreamsBuilderFactoryBean` through this customizer, you can customize the corresponding `KafkaStreams` using `KafkaStreamsCustomzier`. +Both of these customizers are part of the Spring for Apache Kafka project. + +Here is an example of using the `StreamsBuilderFactoryBeanCustomizer`. + +``` +@Bean +public StreamsBuilderFactoryBeanCustomizer streamsBuilderFactoryBeanCustomizer() { + return sfb -> sfb.setStateListener((newState, oldState) -> { + //Do some action here! + }); +} +``` + +The above is shown as an illustration of the things you can do to customize the `StreamsBuilderFactoryBean`. +You can essentially call any available mutation operations from `StreamsBuilderFactoryBean` to customize it. +This customizer will be invoked by the binder right before the factory bean is started. + +Once you get access to the `StreamsBuilderFactoryBean`, you can also customize the underlying `KafkaStreams` object. +Here is a blueprint for doing so. + +``` +@Bean +public StreamsBuilderFactoryBeanCustomizer streamsBuilderFactoryBeanCustomizer() { + return factoryBean -> { + factoryBean.setKafkaStreamsCustomizer(new KafkaStreamsCustomizer() { + @Override + public void customize(KafkaStreams kafkaStreams) { + kafkaStreams.setUncaughtExceptionHandler((t, e) -> { + + }); + } + }); + }; +} +``` + +`KafkaStreamsCustomizer` will be called by the `StreamsBuilderFactoryBeabn` right before the underlying `KafkaStreams` gets started. + +There can only be one `StreamsBuilderFactoryBeanCustomizer` in the entire application. +Then how do we account for multiple Kafka Streams processors as each of them are backed up by individual `StreamsBuilderFactoryBean` objects? +In that case, if the customization needs to be different for those processors, then the application needs to apply some filter based on the application ID. + +For e.g, + +``` +@Bean +public StreamsBuilderFactoryBeanCustomizer streamsBuilderFactoryBeanCustomizer() { + + return factoryBean -> { + if (factoryBean.getStreamsConfiguration().getProperty(StreamsConfig.APPLICATION_ID_CONFIG) + .equals("processor1-application-id")) { + factoryBean.setKafkaStreamsCustomizer(new KafkaStreamsCustomizer() { + @Override + public void customize(KafkaStreams kafkaStreams) { + kafkaStreams.setUncaughtExceptionHandler((t, e) -> { + + }); + } + }); + } + }; +``` + +==== Using Customizer to register a global state store + +As mentioned above, the binder does not provide a first class way to register global state stores as a feature. +For that, you need to use the customizer. +Here is how that can be done. + +``` +@Bean +public StreamsBuilderFactoryBeanCustomizer customizer() { + return fb -> { + try { + final StreamsBuilder streamsBuilder = fb.getObject(); + streamsBuilder.addGlobalStore(...); + } + catch (Exception e) { + + } + }; +} +``` + +Again, if you have multiple processors, you want to attach the global state store to the right `StreamsBuilder` by filtering out the other `StreamsBuilderFactoryBean` objects using the application id as outlined above. + +==== Using customizer to register a production exception handler + +In the error handling section, we indicated that the binder does not provide a first class way to deal with production exceptions. +Though that is the case, you can still use the `StreamsBuilderFacotryBean` customizer to register production exception handlers. See below. + +``` +@Bean +public StreamsBuilderFactoryBeanCustomizer customizer() { + return fb -> { + fb.getStreamsConfiguration().put(StreamsConfig.DEFAULT_PRODUCTION_EXCEPTION_HANDLER_CLASS_CONFIG, + CustomProductionExceptionHandler.class); + }; +} +``` + +Once again, if you have multiple processors, you may want to set it appropriately against the correct `StreamsBuilderFactoryBean`. +You may also add such production exception handlers using the configuration property (See below for more on that), but this is an option if you choose to go with a programmatic approach. + +=== Timestamp extractor + +Kafka Streams allows you to control the processing of the consumer records based on various notions of timestamp. +By default, Kafka Streams extracts the timestamp metadata embedded in the consumer record. +You can change this default behavior by providing a different `TimestampExtractor` implementation per input binding. +Here are some details on how that can be done. + +``` +@Bean +public Function, + Function, + Function, KStream>>> process() { + return orderStream -> + customers -> + products -> orderStream; +} + +@Bean +public TimestampExtractor timestampExtractor() { + return new WallclockTimestampExtractor(); +} +``` + +Then you set the above `TimestampExtractor` bean name per consumer binding. + +``` +spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.timestampExtractorBeanName=timestampExtractor +spring.cloud.stream.kafka.streams.bindings.process-in-1.consumer.timestampExtractorBeanName=timestampExtractor +spring.cloud.stream.kafka.streams.bindings.process-in-2.consumer.timestampExtractorBeanName=timestampExtractor" +``` + +If you skip an input consumer binding for setting a custom timestamp extractor, that consumer will use the default settings. + +=== Multi binders with Kafka Streams based binders and regular Kafka Binder + +You can have an application where you have both a function/consumer/supplier that is based on the regular Kafka binder and a Kafka Streams based processor. +However, you cannot mix both of them within a single function or consumer. + +Here is an example, where you have both binder based components within the same application. + +``` +@Bean +public Function process() { + return s -> s; +} + +@Bean +public Function, KStream> kstreamProcess() { + + return input -> input; +} + +``` + +This is the relevant parts from the configuration: + +``` +spring.cloud.stream.function.definition=process;kstreamProcess +spring.cloud.stream.bindings.process-in-0.destination=foo +spring.cloud.stream.bindings.process-out-0.destination=bar +spring.cloud.stream.bindings.kstreamProcess-in-0.destination=bar +spring.cloud.stream.bindings.kstreamProcess-out-0.destination=foobar +``` + +Things become a bit more complex if you have the same application as above, but is dealing with two different Kafka clusters, for e.g. the regular `process` is acting upon both Kafka cluster 1 and cluster 2 (receiving data from cluster-1 and sending to cluster-2) and the Kafka Streams processor is acting upon Kafka cluster 2. +Then you have to use the https://cloud.spring.io/spring-cloud-stream/reference/html/spring-cloud-stream.html#multiple-binders[multi binder] facilities provided by Spring Cloud Stream. + +Here is how your configuration may change in that scenario. + +``` +# multi binder configuration +spring.cloud.stream.binders.kafka1.type: kafka +spring.cloud.stream.binders.kafka1.environment.spring.cloud.stream.kafka.streams.binder.brokers=${kafkaCluster-1} #Replace kafkaCluster-1 with the approprate IP of the cluster +spring.cloud.stream.binders.kafka2.type: kafka +spring.cloud.stream.binders.kafka2.environment.spring.cloud.stream.kafka.streams.binder.brokers=${kafkaCluster-2} #Replace kafkaCluster-2 with the approprate IP of the cluster +spring.cloud.stream.binders.kafka3.type: kstream +spring.cloud.stream.binders.kafka3.environment.spring.cloud.stream.kafka.streams.binder.brokers=${kafkaCluster-2} #Replace kafkaCluster-2 with the approprate IP of the cluster + +spring.cloud.stream.function.definition=process;kstreamProcess + +# From cluster 1 to cluster 2 with regular process function +spring.cloud.stream.bindings.process-in-0.destination=foo +spring.cloud.stream.bindings.process-in-0.binder=kafka1 # source from cluster 1 +spring.cloud.stream.bindings.process-out-0.destination=bar +spring.cloud.stream.bindings.process-out-0.binder=kafka2 # send to cluster 2 + +# Kafka Streams processor on cluster 2 +spring.cloud.stream.bindings.kstreamProcess-in-0.destination=bar +spring.cloud.stream.bindings.kstreamProcess-in-0.binder=kafka3 +spring.cloud.stream.bindings.kstreamProcess-out-0.destination=foobar +spring.cloud.stream.bindings.kstreamProcess-out-0.binder=kafka3 +``` + +Pay attention to the above configuration. +We have two kinds of binders, but 3 binders all in all, first one is the regular Kafka binder based on cluster 1 (`kafka1`), then another Kafka binder based on cluster 2 (`kafka2`) and finally the `kstream` one (`kafka3`). +The first processor in the application receives data from `kafka1` and publishes to `kafka2` where both binders are based on regular Kafka binder but differnt clusters. +The second processor, which is a Kafka Streams processor consumes data from `kafka3` which is the same cluster as `kafka2`, but a different binder type. + +Since there are three different binder types available in the Kafka Streams family of binders - `kstream`, `ktable` and `globalktable` - if your application has multiple bindings based on any of these binders, that needs to be explicitly provided as the binder type. + +For e.g if you have a processor as below, + +``` +@Bean +public Function, + Function, + Function, KStream>>> enrichOrder() { + + ... +} +``` + +then, this has to be configured in a multi binder scenario as the following. +Please note that this is only needed if you have a true multi-binder scenario where there are multiple processors dealing with multiple clusters within a single application. +In that case, the binders need to be explicitly provided with the bindings to distinguish from other processor's binder types and clusters. + +``` +spring.cloud.stream.binders.kafka1.type: kstream +spring.cloud.stream.binders.kafka1.environment.spring.cloud.stream.kafka.streams.binder.brokers=${kafkaCluster-2} +spring.cloud.stream.binders.kafka2.type: ktable +spring.cloud.stream.binders.kafka2.environment.spring.cloud.stream.kafka.streams.binder.brokers=${kafkaCluster-2} +spring.cloud.stream.binders.kafka3.type: globalktable +spring.cloud.stream.binders.kafka3.environment.spring.cloud.stream.kafka.streams.binder.brokers=${kafkaCluster-2} + +spring.cloud.stream.bindings.enrichOrder-in-0.binder=kafka1 #kstream +spring.cloud.stream.bindings.enrichOrder-in-1.binder=kafka2 #ktablr +spring.cloud.stream.bindings.enrichOrder-in-2.binder=kafka3 #globalktable +spring.cloud.stream.bindings.enrichOrder-out-0.binder=kafka1 #kstream + +# rest of the configuration is omitted. + +``` + +=== State Cleanup + +By default, no local state is cleaned up when the binding is stopped. +This is the same behavior effective from Spring Kafka version 2.7. +See https://docs.spring.io/spring-kafka/reference/html/#streams-config[Spring Kafka documentation] for more details. +To modify this behavior simply add a single `CleanupConfig` `@Bean` (configured to clean up on start, stop, or neither) to the application context; the bean will be detected and wired into the factory bean. + +=== Kafka Streams topology visualization + +Kafka Streams binder provides the following actuator endpoints for retrieving the topology description using which you can visualize the topology using external tools. + +`/actuator/kafkastreamstopology` + +`/actuator/kafkastreamstopology/` + +You need to include the actuator and web dependencies from Spring Boot to access these endpoints. +Further, you also need to add `kafkastreamstopology` to `management.endpoints.web.exposure.include` property. +By default, the `kafkastreamstopology` endpoint is disabled. + +=== Event type based routing in Kafka Streams applications + +Routing functions available in regular message channel based binders are not supported in Kafka Streams binder. +However, Kafka Streams binder still provides routing capabilities through the event type record header on the inbound records. + +To enable routing based on event types, the application must provide the following property. + +`spring.cloud.stream.kafka.streams.bindings..consumer.eventTypes`. + +This can be a comma separated value. + +For example, lets assume we have this function: + +``` +@Bean +public Function, KStream> process() { + return input -> input; +} +``` + +Let us also assume that we only want the business logic in this function to be executed, if the incoming record has event types as `foo` or `bar`. +That can be expressed as below using the `eventTypes` property on the binding. + +`spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.eventTypes=foo,bar` + +Now, when the application runs, the binder checks each incoming records for the header `event_type` and see if it has value set as `foo` or `bar`. +If it does not find either of them, then the function execution will be skipped. + +By default, the binder expects the record header key to be `event_type`, but that can be changed per binding. +For instance, if we want to change the header key on this binding to `my_event` instead of the default, that can be changed as below. + +`spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.eventTypeHeaderKey=my_event`. + +=== Binding visualization and control in Kafka Streams binder + +Starting with version 3.1.2, Kafka Streams binder supports binding visualization and control. +The only two lifecycle phases supported are `STOPPED` and `STARTED`. +The lifecycle phases `PAUSED` and `RESUMED` are not available in Kafka Streams binder. + +In order to activate binding visualization and control, the application needs to include the following two dependencies. + +``` + + org.springframework.boot + spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-starter-web + +``` + +If you prefer using webflux, you can then include `spring-boot-starter-webflux` instead of the standard web dependency. + +In addition, you also need to set the following property: + +``` +management.endpoints.web.exposure.include=bindings +``` + +To illustrate this feature further, let us use the following application as a guide: + +``` +@SpringBootApplication +public class KafkaStreamsApplication { + + public static void main(String[] args) { + SpringApplication.run(KafkaStreamsApplication.class, args); + } + + @Bean + public Consumer> consumer() { + return s -> s.foreach((key, value) -> System.out.println(value)); + } + + @Bean + public Function, KStream> function() { + return ks -> ks; + } + +} +``` + +As we can see, the application has two Kafka Streams functions - one, a consumer and another a function. +The consumer binding is named by default as `consumer-in-0`. +Similarly, for the function, the input binding is `function-in-0` and the output binding is `function-out-0`. + +Once the application is started, we can find details about the bindings using the following bindings endpoint. + +``` + curl http://localhost:8080/actuator/bindings | jq . +[ + { + "bindingName": "consumer-in-0", + "name": "consumer-in-0", + "group": "consumer-applicationId", + "pausable": false, + "state": "running", + "paused": false, + "input": true, + "extendedInfo": {} + }, + { + "bindingName": "function-in-0", + "name": "function-in-0", + "group": "function-applicationId", + "pausable": false, + "state": "running", + "paused": false, + "input": true, + "extendedInfo": {} + }, + { + "bindingName": "function-out-0", + "name": "function-out-0", + "group": "function-applicationId", + "pausable": false, + "state": "running", + "paused": false, + "input": false, + "extendedInfo": {} + } +] +``` + +The details about all three bindings can be found above. + +Let us now stop the consumer-in-0 binding. + +``` +curl -d '{"state":"STOPPED"}' -H "Content-Type: application/json" -X POST http://localhost:8080/actuator/bindings/consumer-in-0 +``` + +At this point, no records will be received through this binding. + +Start the binding again. + +``` +curl -d '{"state":"STARTED"}' -H "Content-Type: application/json" -X POST http://localhost:8080/actuator/bindings/consumer-in-0 +``` + +When there are multiple bindings present on a single function, invoking these operations on any of those bindings will work. +This is because all the bindings on a single function are backed by the same `StreamsBuilderFactoryBean`. +Therefore, for the function above, either `function-in-0` or `function-out-0` will work. + +=== Manually starting Kafka Streams processors + +Spring Cloud Stream Kafka Streams binder offers an abstraction called `StreamsBuilderFactoryManager` on top of the `StreamsBuilderFactoryBean` from Spring for Apache Kafka. +This manager API is used for controlling the multiple `StreamsBuilderFactoryBean` per processor in a binder based application. +Therefore, when using the binder, if you manually want to control the auto starting of the various `StreamsBuilderFactoryBean` objects in the application, you need to use `StreamsBuilderFactoryManager`. +You can use the property `spring.kafka.streams.auto-startup` and set this to `false` in order to turn off auto starting of the processors. +Then, in the application, you can use something as below to start the processors using `StreamsBuilderFactoryManager`. + +``` +@Bean +public ApplicationRunner runner(StreamsBuilderFactoryManager sbfm) { + return args -> { + sbfm.start(); + }; +} +``` + +This feature is handy, when you want your application to start in the main thread and let Kafka Streams processors start separately. +For example, when you have a large state store that needs to be restored, if the processors are started normally as is the default case, this may block your application to start. +If you are using some sort of liveness probe mechanism (for example on Kubernetes), it may think that the application is down and attempt a restart. +In order to correct this, you can set `spring.kafka.streams.auto-startup` to `false` and follow the approach above. + +Keep in mind that, when using the Spring Cloud Stream binder, you are not directly dealing with `StreamsBuilderFactoryBean` from Spring for Apache Kafka, rather `StreamsBuilderFactoryManager`, as the `StreamsBuilderFactoryBean` objects are internally managed by the binder. + +=== Manually starting Kafka Streams processors selectively + +While the approach laid out above will unconditionally apply auto start `false` to all the Kafka Streams processors in the application through `StreamsBuilderFactoryManager`, it is often desirable that only individually selected Kafka Streams processors are not auto started. +For instance, let us assume that you have three different functions (processors) in your application and for one of the processors, you do not want to start it as part of the application startup. +Here is an example of such a situation. + +``` + +@Bean +public Function, KStream> process1() { + +} + +@Bean +public Consumer> process2() { + +} + +@Bean +public BiFunction, KTable, KStream> process3() { + +} + +``` + +In this scenario above, if you set `spring.kafka.streams.auto-startup` to `false`, then none of the processors will auto start during the application startup. +In that case, you have to programmatically start them as described above by calling `start()` on the underlying `StreamsBuilderFactoryManager`. +However, if we have a use case to selectively disable only one processor, then you have to set `auto-startup` on the individual binding for that processor. +Let us assume that we don't want our `process3` function to auto start. +This is a `BiFunction` with two input bindings - `process3-in-0` and `process3-in-1`. +In order to avoid auto start for this processor, you can pick any of these input bindings and set `auto-startup` on them. +It does not matter which binding you pick; if you wish, you can set `auto-startup` to `false` on both of them, but one will be sufficient. +Because they share the same factory bean, you don't have to set autoStartup to false on both bindings, but it probably makes sense to do so, for clarity. + +Here is the Spring Cloud Stream property that you can use to disable auto startup for this processor. + +``` +spring.cloud.stream.bindings.process3-in-0.consumer.auto-startup: false +``` + +or + +``` +spring.cloud.stream.bindings.process3-in-1.consumer.auto-startup: false +``` + +Then, you can manually start the processor either using the REST endpoint or using the `BindingsEndpoint` API as shown below. +For this, you need to ensure that you have the Spring Boot actuator dependency on the classpath. + +``` +curl -d '{"state":"STARTED"}' -H "Content-Type: application/json" -X POST http://localhost:8080/actuator/bindings/process3-in-0 +``` + +or + +``` +@Autowired +BindingsEndpoint endpoint; + +@Bean +public ApplicationRunner runner() { + return args -> { + endpoint.changeState("process3-in-0", State.STARTED); + }; +} +``` + +See https://docs.spring.io/spring-cloud-stream/docs/current/reference/html/spring-cloud-stream.html#binding_visualization_control[this section] from the reference docs for more details on this mechanism. + +NOTE: When controlling the bindings by disabling `auto-startup` as described in this section, please note that this is only available for consumer bindings. +In other words, if you use the producer binding, `process3-out-0`, that does not have any effect in terms of disabling the auto starting of the processor, although this producer binding uses the same `StreamsBuilderFactoryBean` as the consumer bindings. + +=== Tracing using Spring Cloud Sleuth + +When Spring Cloud Sleuth is on the classpath of a Spring Cloud Stream Kafka Streams binder based application, both its consumer and producer are automatically instrumented with tracing information. +However, in order to trace any application specific operations, those need to be explicitly instrumented by the user code. +This can be done by injecting the `KafkaStreamsTracing` bean from Spring Cloud Sleuth in the application and then invoke various Kafka Streams operations through this injected bean. +Here are some examples of using it. + +``` +@Bean +public BiFunction, KTable, KStream> clicks(KafkaStreamsTracing kafkaStreamsTracing) { + return (userClicksStream, userRegionsTable) -> (userClicksStream + .transformValues(kafkaStreamsTracing.peek("span-1", (key, value) -> LOG.info("key/value: " + key + "/" + value))) + .leftJoin(userRegionsTable, (clicks, region) -> new RegionWithClicks(region == null ? + "UNKNOWN" : region, clicks), + Joined.with(Serdes.String(), Serdes.Long(), null)) + .transform(kafkaStreamsTracing.map("span-2", (key, value) -> { + LOG.info("Click Info: " + value.getRegion() + "/" + value.getClicks()); + return new KeyValue<>(value.getRegion(), + value.getClicks()); + })) + .groupByKey(Grouped.with(Serdes.String(), Serdes.Long())) + .reduce(Long::sum, Materialized.as(CLICK_UPDATES)) + .toStream()); +} +``` + +In the example above, there are two places where it adds explicit tracing instrumentation. +First, we are logging the key/value information from the incoming `KStream`. +When this information is logged, the associated span and trace IDs get logged as well so that a monitoring system can track them and correlate with the same span id. +Second, when we call a `map` operation, instead of calling it directly on the `KStream` class, we wrap it inside a `transform` operation and then call `map` from `KafkaStreamsTracing`. +In this case also, the logged message will contain the span ID and trace ID. + +Here is another example, where we use the low-level transformer API for accessing the various Kafka Streams headers. +When spring-cloud-sleuth is on the classpath, all the tracing headers can also be accessed like this. + +``` +@Bean +public Function, KStream> process(KafkaStreamsTracing kafkaStreamsTracing) { + return input -> input.transform(kafkaStreamsTracing.transformer( + "transformer-1", + () -> new Transformer>() { + ProcessorContext context; + + @Override + public void init(ProcessorContext context) { + this.context = context; + } + + @Override + public KeyValue transform(String key, String value) { + LOG.info("Headers: " + this.context.headers()); + LOG.info("K/V:" + key + "/" + value); + // More transformations, business logic execution, etc. go here. + return KeyValue.pair(key, value); + } + + @Override + public void close() { + } + })); +} +``` + +=== Configuration Options + +This section contains the configuration options used by the Kafka Streams binder. + +For common configuration options and properties pertaining to binder, refer to the <>. + +==== Kafka Streams Binder Properties + +The following properties are available at the binder level and must be prefixed with `spring.cloud.stream.kafka.streams.binder.` +Any Kafka binder provided properties re-used in Kafka Streams binder must be prefixed with `spring.cloud.stream.kafka.streams.binder` instead of `spring.cloud.stream.kafka.binder`. +The only exception to this rule is when defining the Kafka bootstrap server property in which case either prefix works. + +configuration:: +Map with a key/value pair containing properties pertaining to Apache Kafka Streams API. +This property must be prefixed with `spring.cloud.stream.kafka.streams.binder.`. +Following are some examples of using this property. + +[source] +---- +spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde=org.apache.kafka.common.serialization.Serdes$StringSerde +spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde=org.apache.kafka.common.serialization.Serdes$StringSerde +spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000 +---- + +For more information about all the properties that may go into streams configuration, see `StreamsConfig` JavaDocs in Apache Kafka Streams docs. +All configuration that you can set from `StreamsConfig` can be set through this. +When using this property, it is applicable against the entire application since this is a binder level property. +If you have more than processors in the application, all of them will acquire these properties. +In the case of properties like `application.id`, this will become problematic and therefore you have to carefully examine how the properties from `StreamsConfig` are mapped using this binder level `configuration` property. + +functions..applicationId:: +Applicable only for functional style processors. +This can be used for setting application ID per function in the application. +In the case of multiple functions, this is a handy way to set the application ID. + +functions..configuration:: +Applicable only for functional style processors. +Map with a key/value pair containing properties pertaining to Apache Kafka Streams API. +This is similar to the binder level `configuration` property describe above, but this level of `configuration` property is restricted only against the named function. +When you have multiple processors and you want to restrict access to the configuration based on particular functions, you might want to use this. +All `StreamsConfig` properties can be used here. + +brokers:: +Broker URL ++ +Default: `localhost` +zkNodes:: +Zookeeper URL ++ +Default: `localhost` + +deserializationExceptionHandler:: +Deserialization error handler type. +This handler is applied at the binder level and thus applied against all input binding in the application. +There is a way to control it in a more fine-grained way at the consumer binding level. +Possible values are - `logAndContinue`, `logAndFail`, `skipAndContinue` or `sendToDlq` ++ +Default: `logAndFail` + +applicationId:: +Convenient way to set the application.id for the Kafka Streams application globally at the binder level. +If the application contains multiple functions or `StreamListener` methods, then the application id should be set differently. +See above where setting the application id is discussed in detail. ++ +Default: application will generate a static application ID. See the application ID section for more details. + +stateStoreRetry.maxAttempts:: +Max attempts for trying to connect to a state store. ++ +Default: 1 + +stateStoreRetry.backoffPeriod:: +Backoff period when trying to connect to a state store on a retry. ++ +Default: 1000 ms + +consumerProperties:: +Arbitrary consumer properties at the binder level. + +producerProperties:: +Arbitrary producer properties at the binder level. + +includeStoppedProcessorsForHealthCheck:: +When bindings for processors are stopped through actuator, then this processor will not participate in the health check by default. +Set this property to `true` to enable health check for all processors including the ones that are currently stopped through bindings actuator endpoint. ++ +Default: false + +==== Kafka Streams Producer Properties + +The following properties are _only_ available for Kafka Streams producers and must be prefixed with `spring.cloud.stream.kafka.streams.bindings..producer.` +For convenience, if there are multiple output bindings and they all require a common value, that can be configured by using the prefix `spring.cloud.stream.kafka.streams.default.producer.`. + +keySerde:: +key serde to use ++ +Default: See the above discussion on message de/serialization + +valueSerde:: +value serde to use ++ +Default: See the above discussion on message de/serialization + +useNativeEncoding:: +flag to enable/disable native encoding ++ +Default: `true`. + +streamPartitionerBeanName:: +Custom outbound partitioner bean name to be used at the consumer. +Applications can provide custom `StreamPartitioner` as a Spring bean and the name of this bean can be provided to the producer to use instead of the default one. ++ +Default: See the discussion above on outbound partition support. + +producedAs:: +Custom name for the sink component to which the processor is producing to. ++ +Deafult: `none` (generated by Kafka Streams) + +==== Kafka Streams Consumer Properties + +The following properties are available for Kafka Streams consumers and must be prefixed with `spring.cloud.stream.kafka.streams.bindings..consumer.` +For convenience, if there are multiple input bindings and they all require a common value, that can be configured by using the prefix `spring.cloud.stream.kafka.streams.default.consumer.`. + +applicationId:: +Setting application.id per input binding. This is only preferred for `StreamListener` based processors, for function based processors see other approaches outlined above. ++ +Default: See above. + +keySerde:: +key serde to use ++ +Default: See the above discussion on message de/serialization + +valueSerde:: +value serde to use ++ +Default: See the above discussion on message de/serialization + +materializedAs:: +state store to materialize when using incoming KTable types ++ +Default: `none`. + +useNativeDecoding:: +flag to enable/disable native decoding ++ +Default: `true`. + +dlqName:: +DLQ topic name. ++ +Default: See above on the discussion of error handling and DLQ. + +startOffset:: +Offset to start from if there is no committed offset to consume from. +This is mostly used when the consumer is consuming from a topic for the first time. +Kafka Streams uses `earliest` as the default strategy and the binder uses the same default. +This can be overridden to `latest` using this property. ++ +Default: `earliest`. + +Note: Using `resetOffsets` on the consumer does not have any effect on Kafka Streams binder. +Unlike the message channel based binder, Kafka Streams binder does not seek to beginning or end on demand. + +deserializationExceptionHandler:: +Deserialization error handler type. +This handler is applied per consumer binding as opposed to the binder level property described before. +Possible values are - `logAndContinue`, `logAndFail`, `skipAndContinue` or `sendToDlq` ++ +Default: `logAndFail` + +timestampExtractorBeanName:: +Specific time stamp extractor bean name to be used at the consumer. +Applications can provide `TimestampExtractor` as a Spring bean and the name of this bean can be provided to the consumer to use instead of the default one. ++ +Default: See the discussion above on timestamp extractors. + +eventTypes:: +Comma separated list of supported event types for this binding. ++ +Default: `none` + +eventTypeHeaderKey:: +Event type header key on each incoming records through this binding. ++ +Default: `event_type` + +consumedAs:: +Custom name for the source component from which the processor is consuming from. ++ +Deafult: `none` (generated by Kafka Streams) + +==== Special note on concurrency + +In Kafka Streams, you can control of the number of threads a processor can create using the `num.stream.threads` property. +This, you can do using the various `configuration` options described above under binder, functions, producer or consumer level. +You can also use the `concurrency` property that core Spring Cloud Stream provides for this purpose. +When using this, you need to use it on the consumer. +When you have more than one input bindings either in a function or `StreamListener`, set this on the first input binding. +For e.g. when setting `spring.cloud.stream.bindings.process-in-0.consumer.concurrency`, it will be translated as `num.stream.threads` by the binder. +If you have multiple processors and one processor defines binding level concurrency, but not the others, those ones with no binding level concurrency will default back to the binder wide property specified through +`spring.cloud.stream.kafka.streams.binder.configuration.num.stream.threads`. +If this binder configuration is not available, then the application will use the default set by Kafka Streams. diff --git a/k-binder/docs/src/main/asciidoc/overview.adoc b/k-binder/docs/src/main/asciidoc/overview.adoc new file mode 100644 index 000000000..a37e1eff0 --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/overview.adoc @@ -0,0 +1,966 @@ +[partintro] +-- +This guide describes the Apache Kafka implementation of the Spring Cloud Stream Binder. +It contains information about its design, usage, and configuration options, as well as information on how the Stream Cloud Stream concepts map onto Apache Kafka specific constructs. +In addition, this guide explains the Kafka Streams binding capabilities of Spring Cloud Stream. +-- + +== Apache Kafka Binder + +=== Usage + +To use Apache Kafka binder, you need to add `spring-cloud-stream-binder-kafka` as a dependency to your Spring Cloud Stream application, as shown in the following example for Maven: + +[source,xml] +---- + + org.springframework.cloud + spring-cloud-stream-binder-kafka + +---- + +Alternatively, you can also use the Spring Cloud Stream Kafka Starter, as shown in the following example for Maven: + +[source,xml] +---- + + org.springframework.cloud + spring-cloud-starter-stream-kafka + +---- + +=== Overview + +The following image shows a simplified diagram of how the Apache Kafka binder operates: + +.Kafka Binder +image::{github-raw}/docs/src/main/asciidoc/images/kafka-binder.png[width=300,scaledwidth="50%"] + +The Apache Kafka Binder implementation maps each destination to an Apache Kafka topic. +The consumer group maps directly to the same Apache Kafka concept. +Partitioning also maps directly to Apache Kafka partitions as well. + +The binder currently uses the Apache Kafka `kafka-clients` version `2.3.1`. +This client can communicate with older brokers (see the Kafka documentation), but certain features may not be available. +For example, with versions earlier than 0.11.x.x, native headers are not supported. +Also, 0.11.x.x does not support the `autoAddPartitions` property. + +=== Configuration Options + +This section contains the configuration options used by the Apache Kafka binder. + +For common configuration options and properties pertaining to the binder, see the https://cloud.spring.io/spring-cloud-static/spring-cloud-stream/current/reference/html/spring-cloud-stream.html#binding-properties[binding properties] in core documentation. + +==== Kafka Binder Properties + +spring.cloud.stream.kafka.binder.brokers:: +A list of brokers to which the Kafka binder connects. ++ +Default: `localhost`. +spring.cloud.stream.kafka.binder.defaultBrokerPort:: +`brokers` allows hosts specified with or without port information (for example, `host1,host2:port2`). +This sets the default port when no port is configured in the broker list. ++ +Default: `9092`. +spring.cloud.stream.kafka.binder.configuration:: +Key/Value map of client properties (both producers and consumer) passed to all clients created by the binder. +Due to the fact that these properties are used by both producers and consumers, usage should be restricted to common properties -- for example, security settings. +Unknown Kafka producer or consumer properties provided through this configuration are filtered out and not allowed to propagate. +Properties here supersede any properties set in boot. ++ +Default: Empty map. +spring.cloud.stream.kafka.binder.consumerProperties:: +Key/Value map of arbitrary Kafka client consumer properties. +In addition to support known Kafka consumer properties, unknown consumer properties are allowed here as well. +Properties here supersede any properties set in boot and in the `configuration` property above. ++ +Default: Empty map. +spring.cloud.stream.kafka.binder.headers:: +The list of custom headers that are transported by the binder. +Only required when communicating with older applications (<= 1.3.x) with a `kafka-clients` version < 0.11.0.0. Newer versions support headers natively. ++ +Default: empty. +spring.cloud.stream.kafka.binder.healthTimeout:: +The time to wait to get partition information, in seconds. +Health reports as down if this timer expires. ++ +Default: 10. +spring.cloud.stream.kafka.binder.requiredAcks:: +The number of required acks on the broker. +See the Kafka documentation for the producer `acks` property. ++ +Default: `1`. +spring.cloud.stream.kafka.binder.minPartitionCount:: +Effective only if `autoCreateTopics` or `autoAddPartitions` is set. +The global minimum number of partitions that the binder configures on topics on which it produces or consumes data. +It can be superseded by the `partitionCount` setting of the producer or by the value of `instanceCount * concurrency` settings of the producer (if either is larger). ++ +Default: `1`. +spring.cloud.stream.kafka.binder.producerProperties:: +Key/Value map of arbitrary Kafka client producer properties. +In addition to support known Kafka producer properties, unknown producer properties are allowed here as well. +Properties here supersede any properties set in boot and in the `configuration` property above. ++ +Default: Empty map. +spring.cloud.stream.kafka.binder.replicationFactor:: +The replication factor of auto-created topics if `autoCreateTopics` is active. +Can be overridden on each binding. ++ +NOTE: If you are using Kafka broker versions prior to 2.4, then this value should be set to at least `1`. +Starting with version 3.0.8, the binder uses `-1` as the default value, which indicates that the broker 'default.replication.factor' property will be used to determine the number of replicas. +Check with your Kafka broker admins to see if there is a policy in place that requires a minimum replication factor, if that's the case then, typically, the `default.replication.factor` will match that value and `-1` should be used, unless you need a replication factor greater than the minimum. ++ +Default: `-1`. +spring.cloud.stream.kafka.binder.autoCreateTopics:: +If set to `true`, the binder creates new topics automatically. +If set to `false`, the binder relies on the topics being already configured. +In the latter case, if the topics do not exist, the binder fails to start. ++ +NOTE: This setting is independent of the `auto.create.topics.enable` setting of the broker and does not influence it. +If the server is set to auto-create topics, they may be created as part of the metadata retrieval request, with default broker settings. ++ +Default: `true`. +spring.cloud.stream.kafka.binder.autoAddPartitions:: +If set to `true`, the binder creates new partitions if required. +If set to `false`, the binder relies on the partition size of the topic being already configured. +If the partition count of the target topic is smaller than the expected value, the binder fails to start. ++ +Default: `false`. +spring.cloud.stream.kafka.binder.transaction.transactionIdPrefix:: +Enables transactions in the binder. See `transaction.id` in the Kafka documentation and https://docs.spring.io/spring-kafka/reference/html/_reference.html#transactions[Transactions] in the `spring-kafka` documentation. +When transactions are enabled, individual `producer` properties are ignored and all producers use the `spring.cloud.stream.kafka.binder.transaction.producer.*` properties. ++ +Default `null` (no transactions) +spring.cloud.stream.kafka.binder.transaction.producer.*:: +Global producer properties for producers in a transactional binder. +See `spring.cloud.stream.kafka.binder.transaction.transactionIdPrefix` and <> and the general producer properties supported by all binders. ++ +Default: See individual producer properties. + +spring.cloud.stream.kafka.binder.headerMapperBeanName:: +The bean name of a `KafkaHeaderMapper` used for mapping `spring-messaging` headers to and from Kafka headers. +Use this, for example, if you wish to customize the trusted packages in a `BinderHeaderMapper` bean that uses JSON deserialization for the headers. +If this custom `BinderHeaderMapper` bean is not made available to the binder using this property, then the binder will look for a header mapper bean with the name `kafkaBinderHeaderMapper` that is of type `BinderHeaderMapper` before falling back to a default `BinderHeaderMapper` created by the binder. ++ +Default: none. + +spring.cloud.stream.kafka.binder.considerDownWhenAnyPartitionHasNoLeader:: +Flag to set the binder health as `down`, when any partitions on the topic, regardless of the consumer that is receiving data from it, is found without a leader. ++ +Default: `false`. + +spring.cloud.stream.kafka.binder.certificateStoreDirectory:: +When the truststore or keystore certificate location is given as a classpath URL (`classpath:...`), the binder copies the resource from the classpath location inside the JAR file to a location on the filesystem. +This is true for both broker level certificates (`ssl.truststore.location` and `ssl.keystore.location`) and certificates intended for schema registry (`schema.registry.ssl.truststore.location` and `schema.registry.ssl.keystore.location`). +Keep in mind that the truststore and keystore classpath locations must be provided under `spring.cloud.stream.kafka.binder.configuration...`. +For example, `spring.cloud.stream.kafka.binder.configuration.ssl.truststore.location`, ``spring.cloud.stream.kafka.binder.configuration.schema.registry.ssl.truststore.location`, etc. +The file will be moved to the location specified as the value for this property which must be an existing directory on the filesystem that is writable by the process running the application. +If this value is not set and the certificate file is a classpath resource, then it will be moved to System's temp directory as returned by `System.getProperty("java.io.tmpdir")`. +This is also true, if this value is present, but the directory cannot be found on the filesystem or is not writable. ++ +Default: none. + +[[kafka-consumer-properties]] +==== Kafka Consumer Properties + +NOTE: To avoid repetition, Spring Cloud Stream supports setting values for all channels, in the format of `spring.cloud.stream.kafka.default.consumer.=`. + + +The following properties are available for Kafka consumers only and +must be prefixed with `spring.cloud.stream.kafka.bindings..consumer.`. + +admin.configuration:: +Since version 2.1.1, this property is deprecated in favor of `topic.properties`, and support for it will be removed in a future version. + +admin.replicas-assignment:: +Since version 2.1.1, this property is deprecated in favor of `topic.replicas-assignment`, and support for it will be removed in a future version. + +admin.replication-factor:: +Since version 2.1.1, this property is deprecated in favor of `topic.replication-factor`, and support for it will be removed in a future version. + +autoRebalanceEnabled:: +When `true`, topic partitions is automatically rebalanced between the members of a consumer group. +When `false`, each consumer is assigned a fixed set of partitions based on `spring.cloud.stream.instanceCount` and `spring.cloud.stream.instanceIndex`. +This requires both the `spring.cloud.stream.instanceCount` and `spring.cloud.stream.instanceIndex` properties to be set appropriately on each launched instance. +The value of the `spring.cloud.stream.instanceCount` property must typically be greater than 1 in this case. ++ +Default: `true`. +ackEachRecord:: +When `autoCommitOffset` is `true`, this setting dictates whether to commit the offset after each record is processed. +By default, offsets are committed after all records in the batch of records returned by `consumer.poll()` have been processed. +The number of records returned by a poll can be controlled with the `max.poll.records` Kafka property, which is set through the consumer `configuration` property. +Setting this to `true` may cause a degradation in performance, but doing so reduces the likelihood of redelivered records when a failure occurs. +Also, see the binder `requiredAcks` property, which also affects the performance of committing offsets. +This property is deprecated as of 3.1 in favor of using `ackMode`. +If the `ackMode` is not set and batch mode is not enabled, `RECORD` ackMode will be used. ++ +Default: `false`. + +autoCommitOffset:: + +Starting with version 3.1, this property is deprecated. +See `ackMode` for more details on alternatives. +Whether to autocommit offsets when a message has been processed. +If set to `false`, a header with the key `kafka_acknowledgment` of the type `org.springframework.kafka.support.Acknowledgment` header is present in the inbound message. +Applications may use this header for acknowledging messages. +See the examples section for details. +When this property is set to `false`, Kafka binder sets the ack mode to `org.springframework.kafka.listener.AbstractMessageListenerContainer.AckMode.MANUAL` and the application is responsible for acknowledging records. +Also see `ackEachRecord`. ++ +Default: `true`. +ackMode:: +Specify the container ack mode. +This is based on the AckMode enumeration defined in Spring Kafka. +If `ackEachRecord` property is set to `true` and consumer is not in batch mode, then this will use the ack mode of `RECORD`, otherwise, use the provided ack mode using this property. + +autoCommitOnError:: +In pollable consumers, if set to `true`, it always auto commits on error. +If not set (the default) or false, it will not auto commit in pollable consumers. +Note that this property is only applicable for pollable consumers. ++ +Default: not set. +resetOffsets:: +Whether to reset offsets on the consumer to the value provided by startOffset. +Must be false if a `KafkaBindingRebalanceListener` is provided; see <>. +See <> for more information about this property. ++ +Default: `false`. +startOffset:: +The starting offset for new groups. +Allowed values: `earliest` and `latest`. +If the consumer group is set explicitly for the consumer 'binding' (through `spring.cloud.stream.bindings..group`), 'startOffset' is set to `earliest`. Otherwise, it is set to `latest` for the `anonymous` consumer group. +See <> for more information about this property. ++ +Default: null (equivalent to `earliest`). +enableDlq:: +When set to true, it enables DLQ behavior for the consumer. +By default, messages that result in errors are forwarded to a topic named `error..`. +The DLQ topic name can be configurable by setting the `dlqName` property or by defining a `@Bean` of type `DlqDestinationResolver`. +This provides an alternative option to the more common Kafka replay scenario for the case when the number of errors is relatively small and replaying the entire original topic may be too cumbersome. +See <> processing for more information. +Starting with version 2.0, messages sent to the DLQ topic are enhanced with the following headers: `x-original-topic`, `x-exception-message`, and `x-exception-stacktrace` as `byte[]`. +By default, a failed record is sent to the same partition number in the DLQ topic as the original record. +See <> for how to change that behavior. +**Not allowed when `destinationIsPattern` is `true`.** ++ +Default: `false`. +dlqPartitions:: +When `enableDlq` is true, and this property is not set, a dead letter topic with the same number of partitions as the primary topic(s) is created. +Usually, dead-letter records are sent to the same partition in the dead-letter topic as the original record. +This behavior can be changed; see <>. +If this property is set to `1` and there is no `DqlPartitionFunction` bean, all dead-letter records will be written to partition `0`. +If this property is greater than `1`, you **MUST** provide a `DlqPartitionFunction` bean. +Note that the actual partition count is affected by the binder's `minPartitionCount` property. ++ +Default: `none` +configuration:: +Map with a key/value pair containing generic Kafka consumer properties. +In addition to having Kafka consumer properties, other configuration properties can be passed here. +For example some properties needed by the application such as `spring.cloud.stream.kafka.bindings.input.consumer.configuration.foo=bar`. +The `bootstrap.servers` property cannot be set here; use multi-binder support if you need to connect to multiple clusters. ++ +Default: Empty map. +dlqName:: +The name of the DLQ topic to receive the error messages. ++ +Default: null (If not specified, messages that result in errors are forwarded to a topic named `error..`). +dlqProducerProperties:: +Using this, DLQ-specific producer properties can be set. +All the properties available through kafka producer properties can be set through this property. +When native decoding is enabled on the consumer (i.e., useNativeDecoding: true) , the application must provide corresponding key/value serializers for DLQ. +This must be provided in the form of `dlqProducerProperties.configuration.key.serializer` and `dlqProducerProperties.configuration.value.serializer`. ++ +Default: Default Kafka producer properties. +standardHeaders:: +Indicates which standard headers are populated by the inbound channel adapter. +Allowed values: `none`, `id`, `timestamp`, or `both`. +Useful if using native deserialization and the first component to receive a message needs an `id` (such as an aggregator that is configured to use a JDBC message store). ++ +Default: `none` +converterBeanName:: +The name of a bean that implements `RecordMessageConverter`. Used in the inbound channel adapter to replace the default `MessagingMessageConverter`. ++ +Default: `null` +idleEventInterval:: +The interval, in milliseconds, between events indicating that no messages have recently been received. +Use an `ApplicationListener` to receive these events. +See <> for a usage example. ++ +Default: `30000` +destinationIsPattern:: +When true, the destination is treated as a regular expression `Pattern` used to match topic names by the broker. +When true, topics are not provisioned, and `enableDlq` is not allowed, because the binder does not know the topic names during the provisioning phase. +Note, the time taken to detect new topics that match the pattern is controlled by the consumer property `metadata.max.age.ms`, which (at the time of writing) defaults to 300,000ms (5 minutes). +This can be configured using the `configuration` property above. ++ +Default: `false` +topic.properties:: +A `Map` of Kafka topic properties used when provisioning new topics -- for example, `spring.cloud.stream.kafka.bindings.input.consumer.topic.properties.message.format.version=0.9.0.0` ++ +Default: none. +topic.replicas-assignment:: +A Map> of replica assignments, with the key being the partition and the value being the assignments. +Used when provisioning new topics. +See the `NewTopic` Javadocs in the `kafka-clients` jar. ++ +Default: none. +topic.replication-factor:: +The replication factor to use when provisioning topics. Overrides the binder-wide setting. +Ignored if `replicas-assignments` is present. ++ +Default: none (the binder-wide default of -1 is used). +pollTimeout:: +Timeout used for polling in pollable consumers. ++ +Default: 5 seconds. +transactionManager:: +Bean name of a `KafkaAwareTransactionManager` used to override the binder's transaction manager for this binding. +Usually needed if you want to synchronize another transaction with the Kafka transaction, using the `ChainedKafkaTransactionManaager`. +To achieve exactly once consumption and production of records, the consumer and producer bindings must all be configured with the same transaction manager. ++ +Default: none. +txCommitRecovered:: +When using a transactional binder, the offset of a recovered record (e.g. when retries are exhausted and the record is sent to a dead letter topic) will be committed via a new transaction, by default. +Setting this property to `false` suppresses committing the offset of recovered record. ++ +Default: true. +commonErrorHandlerBeanName:: +`CommonErrorHandler` bean name to use per consumer binding. +When present, this user provided `CommonErrorHandler` takes precedence over any other error handlers defined by the binder. +This is a handy way to express error handlers, if the application does not want to use a `ListenerContainerCustomizer` and then check the destination/group combination to set an error handler. ++ +Default: none. + +[[reset-offsets]] +==== Resetting Offsets + +When an application starts, the initial position in each assigned partition depends on two properties `startOffset` and `resetOffsets`. +If `resetOffsets` is `false`, normal Kafka consumer https://kafka.apache.org/documentation/#consumerconfigs_auto.offset.reset[`auto.offset.reset`] semantics apply. +i.e. If there is no committed offset for a partition for the binding's consumer group, the position is `earliest` or `latest`. +By default, bindings with an explicit `group` use `earliest`, and anonymous bindings (with no `group`) use `latest`. +These defaults can be overridden by setting the `startOffset` binding property. +There will be no committed offset(s) the first time the binding is started with a particular `group`. +The other condition where no committed offset exists is if the offset has been expired. +With modern brokers (since 2.1), and default broker properties, the offsets are expired 7 days after the last member leaves the group. +See the https://kafka.apache.org/documentation/#brokerconfigs_offsets.retention.minutes[`offsets.retention.minutes`] broker property for more information. + +When `resetOffsets` is `true`, the binder applies similar semantics to those that apply when there is no committed offset on the broker, as if this binding has never consumed from the topic; i.e. any current committed offset is ignored. + +Following are two use cases when this might be used. + +1. Consuming from a compacted topic containing key/value pairs. +Set `resetOffsets` to `true` and `startOffset` to `earliest`; the binding will perform a `seekToBeginning` on all newly assigned partitions. + +2. Consuming from a topic containing events, where you are only interested in events that occur while this binding is running. +Set `resetOffsets` to `true` and `startOffset` to `latest`; the binding will perform a `seekToEnd` on all newly assigned partitions. + +IMPORTANT: If a rebalance occurs after the initial assignment, the seeks will only be performed on any newly assigned partitions that were not assigned during the initial assignment. + +For more control over topic offsets, see <>; when a listener is provided, `resetOffsets` should not be set to `true`, otherwise, that will cause an error. + +==== Consuming Batches + +Starting with version 3.0, when `spring.cloud.stream.binding..consumer.batch-mode` is set to `true`, all of the records received by polling the Kafka `Consumer` will be presented as a `List` to the listener method. +Otherwise, the method will be called with one record at a time. +The size of the batch is controlled by Kafka consumer properties `max.poll.records`, `fetch.min.bytes`, `fetch.max.wait.ms`; refer to the Kafka documentation for more information. + +Bear in mind that batch mode is not supported with `@StreamListener` - it only works with the newer functional programming model. + +IMPORTANT: Retry within the binder is not supported when using batch mode, so `maxAttempts` will be overridden to 1. +You can configure a `SeekToCurrentBatchErrorHandler` (using a `ListenerContainerCustomizer`) to achieve similar functionality to retry in the binder. +You can also use a manual `AckMode` and call `Ackowledgment.nack(index, sleep)` to commit the offsets for a partial batch and have the remaining records redelivered. +Refer to the https://docs.spring.io/spring-kafka/docs/2.3.0.BUILD-SNAPSHOT/reference/html/#committing-offsets[Spring for Apache Kafka documentation] for more information about these techniques. + +[[kafka-producer-properties]] +==== Kafka Producer Properties + +NOTE: To avoid repetition, Spring Cloud Stream supports setting values for all channels, in the format of `spring.cloud.stream.kafka.default.producer.=`. + + +The following properties are available for Kafka producers only and +must be prefixed with `spring.cloud.stream.kafka.bindings..producer.`. + +admin.configuration:: +Since version 2.1.1, this property is deprecated in favor of `topic.properties`, and support for it will be removed in a future version. + +admin.replicas-assignment:: +Since version 2.1.1, this property is deprecated in favor of `topic.replicas-assignment`, and support for it will be removed in a future version. + +admin.replication-factor:: +Since version 2.1.1, this property is deprecated in favor of `topic.replication-factor`, and support for it will be removed in a future version. + +bufferSize:: +Upper limit, in bytes, of how much data the Kafka producer attempts to batch before sending. ++ +Default: `16384`. +sync:: +Whether the producer is synchronous. ++ +Default: `false`. +sendTimeoutExpression:: +A SpEL expression evaluated against the outgoing message used to evaluate the time to wait for ack when synchronous publish is enabled -- for example, `headers['mySendTimeout']`. +The value of the timeout is in milliseconds. +With versions before 3.0, the payload could not be used unless native encoding was being used because, by the time this expression was evaluated, the payload was already in the form of a `byte[]`. +Now, the expression is evaluated before the payload is converted. ++ +Default: `none`. +batchTimeout:: +How long the producer waits to allow more messages to accumulate in the same batch before sending the messages. +(Normally, the producer does not wait at all and simply sends all the messages that accumulated while the previous send was in progress.) A non-zero value may increase throughput at the expense of latency. ++ +Default: `0`. +messageKeyExpression:: +A SpEL expression evaluated against the outgoing message used to populate the key of the produced Kafka message -- for example, `headers['myKey']`. +With versions before 3.0, the payload could not be used unless native encoding was being used because, by the time this expression was evaluated, the payload was already in the form of a `byte[]`. +Now, the expression is evaluated before the payload is converted. +In the case of a regular processor (`Function` or `Function, Message`), if the produced key needs to be same as the incoming key from the topic, this property can be set as below. +`spring.cloud.stream.kafka.bindings..producer.messageKeyExpression: headers['kafka_receivedMessageKey']` +There is an important caveat to keep in mind for reactive functions. +In that case, it is up to the application to manually copy the headers from the incoming messages to outbound messages. +You can set the header, e.g. `myKey` and use `headers['myKey']` as suggested above or, for convenience, simply set the `KafkaHeaders.MESSAGE_KEY` header, and you do not need to set this property at all. ++ +Default: `none`. +headerPatterns:: +A comma-delimited list of simple patterns to match Spring messaging headers to be mapped to the Kafka `Headers` in the `ProducerRecord`. +Patterns can begin or end with the wildcard character (asterisk). +Patterns can be negated by prefixing with `!`. +Matching stops after the first match (positive or negative). +For example `!ask,as*` will pass `ash` but not `ask`. +`id` and `timestamp` are never mapped. ++ +Default: `*` (all headers - except the `id` and `timestamp`) +configuration:: +Map with a key/value pair containing generic Kafka producer properties. +The `bootstrap.servers` property cannot be set here; use multi-binder support if you need to connect to multiple clusters. ++ +Default: Empty map. +topic.properties:: +A `Map` of Kafka topic properties used when provisioning new topics -- for example, `spring.cloud.stream.kafka.bindings.output.producer.topic.properties.message.format.version=0.9.0.0` ++ +topic.replicas-assignment:: +A Map> of replica assignments, with the key being the partition and the value being the assignments. +Used when provisioning new topics. +See the `NewTopic` Javadocs in the `kafka-clients` jar. ++ +Default: none. +topic.replication-factor:: +The replication factor to use when provisioning topics. Overrides the binder-wide setting. +Ignored if `replicas-assignments` is present. ++ +Default: none (the binder-wide default of -1 is used). +useTopicHeader:: +Set to `true` to override the default binding destination (topic name) with the value of the `KafkaHeaders.TOPIC` message header in the outbound message. +If the header is not present, the default binding destination is used. ++ +Default: `false`. +recordMetadataChannel:: +The bean name of a `MessageChannel` to which successful send results should be sent; the bean must exist in the application context. +The message sent to the channel is the sent message (after conversion, if any) with an additional header `KafkaHeaders.RECORD_METADATA`. +The header contains a `RecordMetadata` object provided by the Kafka client; it includes the partition and offset where the record was written in the topic. ++ +`ResultMetadata meta = sendResultMsg.getHeaders().get(KafkaHeaders.RECORD_METADATA, RecordMetadata.class)` ++ +Failed sends go the producer error channel (if configured); see <>. ++ +Default: null. + +NOTE: The Kafka binder uses the `partitionCount` setting of the producer as a hint to create a topic with the given partition count (in conjunction with the `minPartitionCount`, the maximum of the two being the value being used). +Exercise caution when configuring both `minPartitionCount` for a binder and `partitionCount` for an application, as the larger value is used. +If a topic already exists with a smaller partition count and `autoAddPartitions` is disabled (the default), the binder fails to start. +If a topic already exists with a smaller partition count and `autoAddPartitions` is enabled, new partitions are added. +If a topic already exists with a larger number of partitions than the maximum of (`minPartitionCount` or `partitionCount`), the existing partition count is used. + +compression:: +Set the `compression.type` producer property. +Supported values are `none`, `gzip`, `snappy`, `lz4` and `zstd`. +If you override the `kafka-clients` jar to 2.1.0 (or later), as discussed in the https://docs.spring.io/spring-kafka/docs/2.2.x/reference/html/deps-for-21x.html[Spring for Apache Kafka documentation], and wish to use `zstd` compression, use `spring.cloud.stream.kafka.bindings..producer.configuration.compression.type=zstd`. ++ +Default: `none`. +transactionManager:: +Bean name of a `KafkaAwareTransactionManager` used to override the binder's transaction manager for this binding. +Usually needed if you want to synchronize another transaction with the Kafka transaction, using the `ChainedKafkaTransactionManaager`. +To achieve exactly once consumption and production of records, the consumer and producer bindings must all be configured with the same transaction manager. ++ +Default: none. + +closeTimeout:: +Timeout in number of seconds to wait for when closing the producer. ++ +Default: `30` + +allowNonTransactional:: +Normally, all output bindings associated with a transactional binder will publish in a new transaction, if one is not already in process. +This property allows you to override that behavior. +If set to true, records published to this output binding will not be run in a transaction, unless one is already in process. ++ +Default: `false` + +==== Usage examples + +In this section, we show the use of the preceding properties for specific scenarios. + +===== Example: Setting `ackMode` to `MANUAL` and Relying on Manual Acknowledgement + +This example illustrates how one may manually acknowledge offsets in a consumer application. + +This example requires that `spring.cloud.stream.kafka.bindings.input.consumer.ackMode` be set to `MANUAL`. +Use the corresponding input channel name for your example. + +[source] +---- +@SpringBootApplication +@EnableBinding(Sink.class) +public class ManuallyAcknowdledgingConsumer { + + public static void main(String[] args) { + SpringApplication.run(ManuallyAcknowdledgingConsumer.class, args); + } + + @StreamListener(Sink.INPUT) + public void process(Message message) { + Acknowledgment acknowledgment = message.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT, Acknowledgment.class); + if (acknowledgment != null) { + System.out.println("Acknowledgment provided"); + acknowledgment.acknowledge(); + } + } +} +---- + +===== Example: Security Configuration + +Apache Kafka 0.9 supports secure connections between client and brokers. +To take advantage of this feature, follow the guidelines in the https://kafka.apache.org/090/documentation.html#security_configclients[Apache Kafka Documentation] as well as the Kafka 0.9 https://docs.confluent.io/2.0.0/kafka/security.html[security guidelines from the Confluent documentation]. +Use the `spring.cloud.stream.kafka.binder.configuration` option to set security properties for all clients created by the binder. + +For example, to set `security.protocol` to `SASL_SSL`, set the following property: + +[source] +---- +spring.cloud.stream.kafka.binder.configuration.security.protocol=SASL_SSL +---- + +All the other security properties can be set in a similar manner. + +When using Kerberos, follow the instructions in the https://kafka.apache.org/090/documentation.html#security_sasl_clientconfig[reference documentation] for creating and referencing the JAAS configuration. + +Spring Cloud Stream supports passing JAAS configuration information to the application by using a JAAS configuration file and using Spring Boot properties. + +====== Using JAAS Configuration Files + +The JAAS and (optionally) krb5 file locations can be set for Spring Cloud Stream applications by using system properties. +The following example shows how to launch a Spring Cloud Stream application with SASL and Kerberos by using a JAAS configuration file: + +[source,bash] +---- + java -Djava.security.auth.login.config=/path.to/kafka_client_jaas.conf -jar log.jar \ + --spring.cloud.stream.kafka.binder.brokers=secure.server:9092 \ + --spring.cloud.stream.bindings.input.destination=stream.ticktock \ + --spring.cloud.stream.kafka.binder.configuration.security.protocol=SASL_PLAINTEXT +---- + +====== Using Spring Boot Properties + +As an alternative to having a JAAS configuration file, Spring Cloud Stream provides a mechanism for setting up the JAAS configuration for Spring Cloud Stream applications by using Spring Boot properties. + +The following properties can be used to configure the login context of the Kafka client: + +spring.cloud.stream.kafka.binder.jaas.loginModule:: +The login module name. Not necessary to be set in normal cases. ++ +Default: `com.sun.security.auth.module.Krb5LoginModule`. +spring.cloud.stream.kafka.binder.jaas.controlFlag:: +The control flag of the login module. ++ +Default: `required`. +spring.cloud.stream.kafka.binder.jaas.options:: +Map with a key/value pair containing the login module options. ++ +Default: Empty map. + +The following example shows how to launch a Spring Cloud Stream application with SASL and Kerberos by using Spring Boot configuration properties: + +[source,bash] +---- + java --spring.cloud.stream.kafka.binder.brokers=secure.server:9092 \ + --spring.cloud.stream.bindings.input.destination=stream.ticktock \ + --spring.cloud.stream.kafka.binder.autoCreateTopics=false \ + --spring.cloud.stream.kafka.binder.configuration.security.protocol=SASL_PLAINTEXT \ + --spring.cloud.stream.kafka.binder.jaas.options.useKeyTab=true \ + --spring.cloud.stream.kafka.binder.jaas.options.storeKey=true \ + --spring.cloud.stream.kafka.binder.jaas.options.keyTab=/etc/security/keytabs/kafka_client.keytab \ + --spring.cloud.stream.kafka.binder.jaas.options.principal=kafka-client-1@EXAMPLE.COM +---- + +The preceding example represents the equivalent of the following JAAS file: + +[source] +---- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + keyTab="/etc/security/keytabs/kafka_client.keytab" + principal="kafka-client-1@EXAMPLE.COM"; +}; +---- + +If the topics required already exist on the broker or will be created by an administrator, autocreation can be turned off and only client JAAS properties need to be sent. + +NOTE: Do not mix JAAS configuration files and Spring Boot properties in the same application. +If the `-Djava.security.auth.login.config` system property is already present, Spring Cloud Stream ignores the Spring Boot properties. + +NOTE: Be careful when using the `autoCreateTopics` and `autoAddPartitions` with Kerberos. +Usually, applications may use principals that do not have administrative rights in Kafka and Zookeeper. +Consequently, relying on Spring Cloud Stream to create/modify topics may fail. +In secure environments, we strongly recommend creating topics and managing ACLs administratively by using Kafka tooling. + +====== Multi-binder configuration and JAAS + +When connecting to multiple clusters in which each one requires separate JAAS configuration, then set the JAAS configuration using the property `sasl.jaas.config`. +When this property is present in the applicaiton, it takes precedence over the other strategies mentioned above. +See this https://cwiki.apache.org/confluence/display/KAFKA/KIP-85%3A+Dynamic+JAAS+configuration+for+Kafka+clients[KIP-85] for more details. + +For example, if you have two clusters in your application with separate JAAS configuration, then the following is a template that you can use: + +``` +spring.cloud.stream: + binders: + kafka1: + type: kafka + environment: + spring: + cloud: + stream: + kafka: + binder: + brokers: localhost:9092 + configuration.sasl.jaas.config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"admin-secret\";" + kafka2: + type: kafka + environment: + spring: + cloud: + stream: + kafka: + binder: + brokers: localhost:9093 + configuration.sasl.jaas.config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"user1\" password=\"user1-secret\";" + kafka.binder: + configuration: + security.protocol: SASL_PLAINTEXT + sasl.mechanism: PLAIN +``` + +Note that both the Kafka clusters, and the `sasl.jaas.config` values for each of them are different in the above configuration. + +See this https://github.com/spring-cloud/spring-cloud-stream-samples/tree/main/multi-binder-samples/kafka-multi-binder-jaas[sample application] for more details on how to setup and run such an application. + +[[pause-resume]] +===== Example: Pausing and Resuming the Consumer + +If you wish to suspend consumption but not cause a partition rebalance, you can pause and resume the consumer. +This is facilitated by managing the binding lifecycle as shown in **Binding visualization and control** in the Spring Cloud Stream documentation, using `State.PAUSED` and `State.RESUMED`. + +To resume, you can use an `ApplicationListener` (or `@EventListener` method) to receive `ListenerContainerIdleEvent` instances. +The frequency at which events are published is controlled by the `idleEventInterval` property. + +[[kafka-transactional-binder]] +=== Transactional Binder + +Enable transactions by setting `spring.cloud.stream.kafka.binder.transaction.transactionIdPrefix` to a non-empty value, e.g. `tx-`. +When used in a processor application, the consumer starts the transaction; any records sent on the consumer thread participate in the same transaction. +When the listener exits normally, the listener container will send the offset to the transaction and commit it. +A common producer factory is used for all producer bindings configured using `spring.cloud.stream.kafka.binder.transaction.producer.*` properties; individual binding Kafka producer properties are ignored. + +IMPORTANT: Normal binder retries (and dead lettering) are not supported with transactions because the retries will run in the original transaction, which may be rolled back and any published records will be rolled back too. +When retries are enabled (the common property `maxAttempts` is greater than zero) the retry properties are used to configure a `DefaultAfterRollbackProcessor` to enable retries at the container level. +Similarly, instead of publishing dead-letter records within the transaction, this functionality is moved to the listener container, again via the `DefaultAfterRollbackProcessor` which runs after the main transaction has rolled back. + +If you wish to use transactions in a source application, or from some arbitrary thread for producer-only transaction (e.g. `@Scheduled` method), you must get a reference to the transactional producer factory and define a `KafkaTransactionManager` bean using it. + +==== +[source, java] +---- +@Bean +public PlatformTransactionManager transactionManager(BinderFactory binders, + @Value("${unique.tx.id.per.instance}") String txId) { + + ProducerFactory pf = ((KafkaMessageChannelBinder) binders.getBinder(null, + MessageChannel.class)).getTransactionalProducerFactory(); + KafkaTransactionManager tm = new KafkaTransactionManager<>(pf); + tm.setTransactionId(txId) + return tm; +} +---- +==== + +Notice that we get a reference to the binder using the `BinderFactory`; use `null` in the first argument when there is only one binder configured. +If more than one binder is configured, use the binder name to get the reference. +Once we have a reference to the binder, we can obtain a reference to the `ProducerFactory` and create a transaction manager. + +Then you would use normal Spring transaction support, e.g. `TransactionTemplate` or `@Transactional`, for example: + +==== +[source, java] +---- +public static class Sender { + + @Transactional + public void doInTransaction(MessageChannel output, List stuffToSend) { + stuffToSend.forEach(stuff -> output.send(new GenericMessage<>(stuff))); + } + +} +---- +==== + +If you wish to synchronize producer-only transactions with those from some other transaction manager, use a `ChainedTransactionManager`. + +IMPORTANT: If you deploy multiple instances of your application, each instance needs a unique `transactionIdPrefix`. + +[[kafka-error-channels]] +=== Error Channels + +Starting with version 1.3, the binder unconditionally sends exceptions to an error channel for each consumer destination and can also be configured to send async producer send failures to an error channel. +See https://cloud.spring.io/spring-cloud-static/spring-cloud-stream/current/reference/html/spring-cloud-stream.html#spring-cloud-stream-overview-error-handling[this section on error handling] for more information. + +The payload of the `ErrorMessage` for a send failure is a `KafkaSendFailureException` with properties: + +* `failedMessage`: The Spring Messaging `Message` that failed to be sent. +* `record`: The raw `ProducerRecord` that was created from the `failedMessage` + +There is no automatic handling of producer exceptions (such as sending to a <>). +You can consume these exceptions with your own Spring Integration flow. + +[[kafka-metrics]] +=== Kafka Metrics + +Kafka binder module exposes the following metrics: + +`spring.cloud.stream.binder.kafka.offset`: This metric indicates how many messages have not been yet consumed from a given binder's topic by a given consumer group. +The metrics provided are based on the Micrometer library. +The binder creates the `KafkaBinderMetrics` bean if Micrometer is on the classpath and no other such beans provided by the application. +The metric contains the consumer group information, topic and the actual lag in committed offset from the latest offset on the topic. +This metric is particularly useful for providing auto-scaling feedback to a PaaS platform. + +You can exclude `KafkaBinderMetrics` from creating the necessary infrastructure like consumers and then reporting the metrics by providing the following component in the application. + +``` +@Component +class NoOpBindingMeters { + NoOpBindingMeters(MeterRegistry registry) { + registry.config().meterFilter( + MeterFilter.denyNameStartsWith(KafkaBinderMetrics.OFFSET_LAG_METRIC_NAME)); + } +} +``` + +More details on how to suppress meters selectively can be found https://micrometer.io/docs/concepts#_meter_filters[here]. + +[[kafka-tombstones]] +=== Tombstone Records (null record values) + +When using compacted topics, a record with a `null` value (also called a tombstone record) represents the deletion of a key. +To receive such messages in a `@StreamListener` method, the parameter must be marked as not required to receive a `null` value argument. + +==== +[source, java] +---- +@StreamListener(Sink.INPUT) +public void in(@Header(KafkaHeaders.RECEIVED_MESSAGE_KEY) byte[] key, + @Payload(required = false) Customer customer) { + // customer is null if a tombstone record + ... +} +---- +==== + +[[rebalance-listener]] +=== Using a KafkaBindingRebalanceListener + +Applications may wish to seek topics/partitions to arbitrary offsets when the partitions are initially assigned, or perform other operations on the consumer. +Starting with version 2.1, if you provide a single `KafkaBindingRebalanceListener` bean in the application context, it will be wired into all Kafka consumer bindings. + +==== +[source, java] +---- +public interface KafkaBindingRebalanceListener { + + /** + * Invoked by the container before any pending offsets are committed. + * @param bindingName the name of the binding. + * @param consumer the consumer. + * @param partitions the partitions. + */ + default void onPartitionsRevokedBeforeCommit(String bindingName, Consumer consumer, + Collection partitions) { + + } + + /** + * Invoked by the container after any pending offsets are committed. + * @param bindingName the name of the binding. + * @param consumer the consumer. + * @param partitions the partitions. + */ + default void onPartitionsRevokedAfterCommit(String bindingName, Consumer consumer, Collection partitions) { + + } + + /** + * Invoked when partitions are initially assigned or after a rebalance. + * Applications might only want to perform seek operations on an initial assignment. + * @param bindingName the name of the binding. + * @param consumer the consumer. + * @param partitions the partitions. + * @param initial true if this is the initial assignment. + */ + default void onPartitionsAssigned(String bindingName, Consumer consumer, Collection partitions, + boolean initial) { + + } + +} +---- +==== + +You cannot set the `resetOffsets` consumer property to `true` when you provide a rebalance listener. + +[[retry-and-dlq-processing]] +=== Retry and Dead Letter Processing + +By default, when you configure retry (e.g. `maxAttemts`) and `enableDlq` in a consumer binding, these functions are performed within the binder, with no participation by the listener container or Kafka consumer. + +There are situations where it is preferable to move this functionality to the listener container, such as: + +* The aggregate of retries and delays will exceed the consumer's `max.poll.interval.ms` property, potentially causing a partition rebalance. +* You wish to publish the dead letter to a different Kafka cluster. +* You wish to add retry listeners to the error handler. +* ... + +To configure moving this functionality from the binder to the container, define a `@Bean` of type `ListenerContainerWithDlqAndRetryCustomizer`. +This interface has the following methods: + +==== +[source, java] +---- +/** + * Configure the container. + * @param container the container. + * @param destinationName the destination name. + * @param group the group. + * @param dlqDestinationResolver a destination resolver for the dead letter topic (if + * enableDlq). + * @param backOff the backOff using retry properties (if configured). + * @see #retryAndDlqInBinding(String, String) + */ +void configure(AbstractMessageListenerContainer container, String destinationName, String group, + @Nullable BiFunction, Exception, TopicPartition> dlqDestinationResolver, + @Nullable BackOff backOff); + +/** + * Return false to move retries and DLQ from the binding to a customized error handler + * using the retry metadata and/or a {@code DeadLetterPublishingRecoverer} when + * configured via + * {@link #configure(AbstractMessageListenerContainer, String, String, BiFunction, BackOff)}. + * @param destinationName the destination name. + * @param group the group. + * @return true to disable retrie in the binding + */ +default boolean retryAndDlqInBinding(String destinationName, String group) { + return true; +} +---- +==== + +The destination resolver and `BackOff` are created from the binding properties (if configured). +You can then use these to create a custom error handler and dead letter publisher; for example: + +==== +[source, java] +---- +@Bean +ListenerContainerWithDlqAndRetryCustomizer cust(KafkaTemplate template) { + return new ListenerContainerWithDlqAndRetryCustomizer() { + + @Override + public void configure(AbstractMessageListenerContainer container, String destinationName, + String group, + @Nullable BiFunction, Exception, TopicPartition> dlqDestinationResolver, + @Nullable BackOff backOff) { + + if (destinationName.equals("topicWithLongTotalRetryConfig")) { + ConsumerRecordRecoverer dlpr = new DeadLetterPublishingRecoverer(template), + dlqDestinationResolver); + container.setCommonErrorHandler(new DefaultErrorHandler(dlpr, backOff)); + } + } + + @Override + public boolean retryAndDlqInBinding(String destinationName, String group) { + return !destinationName.contains("topicWithLongTotalRetryConfig"); + } + + }; +} +---- +==== + +Now, only a single retry delay needs to be greater than the consumer's `max.poll.interval.ms` property. + +[[consumer-producer-config-customizer]] +=== Customizing Consumer and Producer configuration + +If you want advanced customization of consumer and producer configuration that is used for creating `ConsumerFactory` and `ProducerFactory` in Kafka, +you can implement the following customizers. + +* ConsumerConfigCustomizer +* ProducerConfigCustomizer + +Both of these interfaces provide a way to configure the config map used for consumer and producer properties. +For example, if you want to gain access to a bean that is defined at the application level, you can inject that in the implementation of the `configure` method. +When the binder discovers that these customizers are available as beans, it will invoke the `configure` method right before creating the consumer and producer factories. + +Both of these interfaces also provide access to both the binding and destination names so that they can be accessed while customizing producer and consumer properties. + +[[admin-client-config-customization]] +=== Customizing AdminClient Configuration + +As with consumer and producer config customization above, applications can also customize the configuration for admin clients by providing an `AdminClientConfigCustomizer`. +AdminClientConfigCustomizer's configure method provides access to the admin client properties, using which you can define further customization. +Binder's Kafka topic provisioner gives the highest precedence for the properties given through this customizer. +Here is an example of providing this customizer bean. + +``` +@Bean +public AdminClientConfigCustomizer adminClientConfigCustomizer() { + return props -> { + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL"); + }; +} +``` + +[[custom-kafka-binder-health-indicator]] +=== Custom Kafka Binder Health Indicator + +Kafka binder activates a default health indicator when Spring Boot actuator is on the classpath. +This health indicator checks the health of the binder and any communication issues with the Kafka broker. +If an application wants to disable this default health check implementation and include a custom implementation, then it can provide an implementation for `KafkaBinderHealth` interface. +`KafkaBinderHealth` is a marker interface that extends from `HealthIndicator`. +In the custom implementation, it must provide an implementation for the `health()` method. +The custom implementation must be present in the application configuration as a bean. +When the binder discovers the custom implementation, it will use that instead of the default implementation. +Here is an example of such a custom implementation bean in the application. + +``` +@Bean +public KafkaBinderHealth kafkaBinderHealthIndicator() { + return new KafkaBinderHealth() { + @Override + public Health health() { + // custom implementation details. + } + }; +} +``` diff --git a/k-binder/docs/src/main/asciidoc/partitions.adoc b/k-binder/docs/src/main/asciidoc/partitions.adoc new file mode 100644 index 000000000..178e062da --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/partitions.adoc @@ -0,0 +1,103 @@ +=== Partitioning with the Kafka Binder + +Apache Kafka supports topic partitioning natively. + +Sometimes it is advantageous to send data to specific partitions -- for example, when you want to strictly order message processing (all messages for a particular customer should go to the same partition). + +The following example shows how to configure the producer and consumer side: + +[source, java] +---- +@SpringBootApplication +@EnableBinding(Source.class) +public class KafkaPartitionProducerApplication { + + private static final Random RANDOM = new Random(System.currentTimeMillis()); + + private static final String[] data = new String[] { + "foo1", "bar1", "qux1", + "foo2", "bar2", "qux2", + "foo3", "bar3", "qux3", + "foo4", "bar4", "qux4", + }; + + public static void main(String[] args) { + new SpringApplicationBuilder(KafkaPartitionProducerApplication.class) + .web(false) + .run(args); + } + + @InboundChannelAdapter(channel = Source.OUTPUT, poller = @Poller(fixedRate = "5000")) + public Message generate() { + String value = data[RANDOM.nextInt(data.length)]; + System.out.println("Sending: " + value); + return MessageBuilder.withPayload(value) + .setHeader("partitionKey", value) + .build(); + } + +} +---- + +.application.yml +[source, yaml] +---- +spring: + cloud: + stream: + bindings: + output: + destination: partitioned.topic + producer: + partition-key-expression: headers['partitionKey'] + partition-count: 12 +---- + +IMPORTANT: The topic must be provisioned to have enough partitions to achieve the desired concurrency for all consumer groups. +The above configuration supports up to 12 consumer instances (6 if their `concurrency` is 2, 4 if their concurrency is 3, and so on). +It is generally best to "`over-provision`" the partitions to allow for future increases in consumers or concurrency. + +NOTE: The preceding configuration uses the default partitioning (`key.hashCode() % partitionCount`). +This may or may not provide a suitably balanced algorithm, depending on the key values. +You can override this default by using the `partitionSelectorExpression` or `partitionSelectorClass` properties. + +Since partitions are natively handled by Kafka, no special configuration is needed on the consumer side. +Kafka allocates partitions across the instances. + +The following Spring Boot application listens to a Kafka stream and prints (to the console) the partition ID to which each message goes: + +[source, java] +---- +@SpringBootApplication +@EnableBinding(Sink.class) +public class KafkaPartitionConsumerApplication { + + public static void main(String[] args) { + new SpringApplicationBuilder(KafkaPartitionConsumerApplication.class) + .web(false) + .run(args); + } + + @StreamListener(Sink.INPUT) + public void listen(@Payload String in, @Header(KafkaHeaders.RECEIVED_PARTITION_ID) int partition) { + System.out.println(in + " received from partition " + partition); + } + +} +---- + +.application.yml +[source, yaml] +---- +spring: + cloud: + stream: + bindings: + input: + destination: partitioned.topic + group: myGroup +---- + +You can add instances as needed. +Kafka rebalances the partition allocations. +If the instance count (or `instance count * concurrency`) exceeds the number of partitions, some consumers are idle. diff --git a/k-binder/docs/src/main/asciidoc/spring-cloud-stream-binder-kafka-aggregate.adoc b/k-binder/docs/src/main/asciidoc/spring-cloud-stream-binder-kafka-aggregate.adoc new file mode 100644 index 000000000..3a673ac2c --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/spring-cloud-stream-binder-kafka-aggregate.adoc @@ -0,0 +1,3 @@ +include::overview.adoc[leveloffset=+1] +include::dlq.adoc[leveloffset=+1] +include::partitions.adoc[leveloffset=+1] diff --git a/k-binder/docs/src/main/asciidoc/spring-cloud-stream-binder-kafka.adoc b/k-binder/docs/src/main/asciidoc/spring-cloud-stream-binder-kafka.adoc new file mode 100644 index 000000000..79d871825 --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/spring-cloud-stream-binder-kafka.adoc @@ -0,0 +1,55 @@ +:github-tag: master +:github-repo: spring-cloud/spring-cloud-stream-binder-kafka +:github-raw: https://raw.githubusercontent.com/{github-repo}/{github-tag} +:github-code: https://github.com/{github-repo}/tree/{github-tag} +:toc: left +:toclevels: 8 +:nofooter: +:sectlinks: true + + +[[spring-cloud-stream-binder-kafka-reference]] += Spring Cloud Stream Kafka Binder Reference Guide +Sabby Anandan, Marius Bogoevici, Eric Bottard, Mark Fisher, Ilayaperumal Gopinathan, Gunnar Hillert, Mark Pollack, Patrick Peralta, Glenn Renfro, Thomas Risberg, Dave Syer, David Turanski, Janne Valkealahti, Benjamin Klein, Henryk Konsek, Gary Russell, Arnaud Jardiné, Soby Chacko +:doctype: book +:toc: +:toclevels: 4 +:source-highlighter: prettify +:numbered: +:icons: font +:hide-uri-scheme: +:spring-cloud-stream-binder-kafka-repo: snapshot +:github-tag: master +:spring-cloud-stream-binder-kafka-docs-version: current +:spring-cloud-stream-binder-kafka-docs: https://docs.spring.io/spring-cloud-stream-binder-kafka/docs/{spring-cloud-stream-binder-kafka-docs-version}/reference +:spring-cloud-stream-binder-kafka-docs-current: https://docs.spring.io/spring-cloud-stream-binder-kafka/docs/current-SNAPSHOT/reference/html/ +:github-repo: spring-cloud/spring-cloud-stream-binder-kafka +:github-raw: https://raw.github.com/{github-repo}/{github-tag} +:github-code: https://github.com/{github-repo}/tree/{github-tag} +:github-wiki: https://github.com/{github-repo}/wiki +:github-master-code: https://github.com/{github-repo}/tree/master +:sc-ext: java +// ====================================================================================== + + +*{project-version}* + + += Reference Guide +include::overview.adoc[] + +include::dlq.adoc[] + +include::partitions.adoc[] + +include::kafka-streams.adoc[] + +include::tips.adoc[] + += Appendices +[appendix] +include::building.adoc[] + +include::contributing.adoc[] + +// ====================================================================================== diff --git a/k-binder/docs/src/main/asciidoc/tips.adoc b/k-binder/docs/src/main/asciidoc/tips.adoc new file mode 100644 index 000000000..ee8f954e2 --- /dev/null +++ b/k-binder/docs/src/main/asciidoc/tips.adoc @@ -0,0 +1,865 @@ +== Tips, Tricks and Recipes + +=== Simple DLQ with Kafka + +==== Problem Statement + +As a developer, I want to write a consumer application that processes records from a Kafka topic. +However, if some error occurs in processing, I don't want the application to stop completely. +Instead, I want to send the record in error to a DLT (Dead-Letter-Topic) and then continue processing new records. + +==== Solution + +The solution for this problem is to use the DLQ feature in Spring Cloud Stream. +For the purposes of this discussion, let us assume that the following is our processor function. + +``` +@Bean +public Consumer processData() { + return s -> { + throw new RuntimeException(); + }; +``` + +This is a very trivial function that throws an exception for all the records that it processes, but you can take this function and extend it to any other similar situations. + +In order to send the records in error to a DLT, we need to provide the following configuration. + +``` +spring.cloud.stream: + bindings: + processData-in-0: + group: my-group + destination: input-topic + kafka: + bindings: + processData-in-0: + consumer: + enableDlq: true + dlqName: input-topic-dlq +``` + +In order to activate DLQ, the application must provide a group name. +Anonymous consumers cannot use the DLQ facilities. +We also need to enable DLQ by setting the `enableDLQ` property on the Kafka consumer binding to `true`. +Finally, we can optionally provide the DLT name by providing the `dlqName` on Kafka consumer binding, which otherwise default to `input-topic-dlq.my-group.error` in this case. + +Note that in the example consumer provided above, the type of the payload is `byte[]`. +By default, the DLQ producer in Kafka binder expects the payload of type `byte[]`. +If that is not the case, then we need to provide the configuration for proper serializer. +For example, let us re-write the consumer function as below: + +``` +@Bean +public Consumer processData() { + return s -> { + throw new RuntimeException(); + }; +} +``` + +Now, we need to tell Spring Cloud Stream, how we want to serialize the data when writing to the DLT. +Here is the modified configuration for this scenario: + +``` +spring.cloud.stream: + bindings: + processData-in-0: + group: my-group + destination: input-topic + kafka: + bindings: + processData-in-0: + consumer: + enableDlq: true + dlqName: input-topic-dlq + dlqProducerProperties: + configuration: + value.serializer: org.apache.kafka.common.serialization.StringSerializer + +``` + +=== DLQ with Advanced Retry Options + +==== Problem Statement + +This is similar to the recipe above, but as a developer I would like to configure the way retries are handled. + +==== Solution + +If you followed the above recipe, then you get the default retry options built into the Kafka binder when the processing encounters an error. + +By default, the binder retires for a maximum of 3 attempts with a one second initial delay, 2.0 multiplier with each back off with a max delay of 10 seconds. +You can change all these configurations as below: + +``` +spring.cloud.stream.bindings.processData-in-0.consumer.maxAtttempts +spring.cloud.stream.bindings.processData-in-0.consumer.backOffInitialInterval +spring.cloud.stream.bindings.processData-in-0.consumer.backOffMultipler +spring.cloud.stream.bindings.processData-in-0.consumer.backOffMaxInterval +``` + +If you want, you can also provide a list of retryable exceptions by providing a map of boolean values. +For example, + +``` +spring.cloud.stream.bindings.processData-in-0.consumer.retryableExceptions.java.lang.IllegalStateException=true +spring.cloud.stream.bindings.processData-in-0.consumer.retryableExceptions.java.lang.IllegalArgumentException=false +``` + +By default, any exceptions not listed in the map above will be retried. +If that is not desired, then you can disable that by providing, + +``` +spring.cloud.stream.bindings.processData-in-0.consumer.defaultRetryable=false +``` + +You can also provide your own `RetryTemplate` and mark it as `@StreamRetryTemplate` which will be scanned and used by the binder. +This is useful when you want more sophisticated retry strategies and policies. + +If you have multiple `@StreamRetryTemplate` beans, then you can specify which one your binding wants by using the property, + +``` +spring.cloud.stream.bindings.processData-in-0.consumer.retry-template-name= +``` + +=== Handling Deserialization errors with DLQ + +==== Problem Statement + +I have a processor that encounters a deserilzartion exception in Kafka consumer. +I would expect that the Spring Cloud Stream DLQ mechanism will catch that scenario, but it does not. +How can I handle this? + +==== Solution + +The normal DLQ mechanism offered by Spring Cloud Stream will not help when Kafka consumer throws an irrecoverable deserialization excepion. +This is because, this exception happens even before the consumer's `poll()` method returns. +Spring for Apache Kafka project offers some great ways to help the binder with this situation. +Let us explore those. + +Assuming this is our function: + +``` +@Bean +public Consumer functionName() { + return s -> { + System.out.println(s); + }; +} +``` + +It is a trivial function that takes a `String` parameter. + +We want to bypass the message converters provided by Spring Cloud Stream and want to use native deserializers instead. +In the case of `String` types, it does not make much sense, but for more complex types like AVRO etc. you have to rely on external deserializers and therefore want to delegate the conversion to Kafka. + +Now when the consumer receives the data, let us assume that there is a bad record that causes a deserilziation errror, maybe someone passed an `Integer` instead of a `String` for example. +In that case, if you don't do something in the application, the excption will be propagated through the chain and your application will exit eventually. + +In order to handle this, you can add a `ListenerContainerCustomizer` `@Bean` that configures a `SeekToCurrentErrorHandler`. +This `SeekToCurrentErrorHandler` is configured with a `DeadLetterPublishingRecoverer`. +We also need to configure an `ErrorHandlingDeserializer` for the consumer. +That sounds like a lot of complex things, but in reality, it boils down to these 3 beans in this case. + +``` +@Bean + public ListenerContainerCustomizer> customizer(SeekToCurrentErrorHandler errorHandler) { + return (container, dest, group) -> { + container.setErrorHandler(errorHandler); + }; + } +``` + +``` + @Bean + public SeekToCurrentErrorHandler errorHandler(DeadLetterPublishingRecoverer deadLetterPublishingRecoverer) { + return new SeekToCurrentErrorHandler(deadLetterPublishingRecoverer); + } +``` + +``` + @Bean + public DeadLetterPublishingRecoverer publisher(KafkaOperations bytesTemplate) { + return new DeadLetterPublishingRecoverer(bytesTemplate); + } +``` + +Let us analyze each of them. +The first one is the `ListenerContainerCustomizer` bean that takes a `SeekToCurrentErrorHandler`. +The container is now customized with that particular error handler. +You can learn more about container customization https://docs.spring.io/spring-cloud-stream/docs/current/reference/html/spring-cloud-stream.html#_advanced_consumer_configuration[here]. + +The second bean is the `SeekToCurrentErrorHandler` that is configured with a publishing to a `DLT`. +See https://docs.spring.io/spring-kafka/docs/current/reference/html/#seek-to-current[here] for more details on `SeekToCurrentErrorHandler`. + +The third bean is the `DeadLetterPublishingRecoverer` that is ultimately responsible for sending to the `DLT`. +By default, the `DLT` topic is named as the ORIGINAL_TOPIC_NAME.DLT. +You can change that though. +See the https://docs.spring.io/spring-kafka/docs/current/reference/html/#dead-letters[docs] for more details. + + +We also need to configure an https://docs.spring.io/spring-kafka/docs/current/reference/html/#error-handling-deserializer[ErrorHandlingDeserializer] through application config. + +The `ErrorHandlingDeserializer` delegates to the actual deserializer. +In case of errors, it sets key/value of the record to be null and includes the raw bytes of the message. +It then sets the exception in a header and passes this record to the listener, which then calls the registered error handler. + +Following is the configuration required: + +``` +spring.cloud.stream: + function: + definition: functionName + bindings: + functionName-in-0: + group: group-name + destination: input-topic + consumer: + use-native-decoding: true + kafka: + bindings: + functionName-in-0: + consumer: + enableDlq: true + dlqName: dlq-topic + dlqProducerProperties: + configuration: + value.serializer: org.apache.kafka.common.serialization.StringSerializer + configuration: + value.deserializer: org.springframework.kafka.support.serializer.ErrorHandlingDeserializer + spring.deserializer.value.delegate.class: org.apache.kafka.common.serialization.StringDeserializer +``` + +We are providing the `ErrorHandlingDeserializer` through the `configuration` property on the binding. +We are also indicating that the actual deserializer to delegate is the `StringDeserializer`. + +Keep in mind that none of the dlq properties above are relevant for the discussions in this recipe. +They are purely meant for addressing any application level errors only. + +=== Basic offset management in Kafka binder + +==== Problem Statement + +I want to write a Spring Cloud Stream Kafka consumer applicaiton and not sure about how it manages Kafka consumer offsets. +Can you exaplain? + +==== Solution + +We encourage you read the https://docs.spring.io/spring-cloud-stream-binder-kafka/docs/current/reference/html/spring-cloud-stream-binder-kafka.html#reset-offsets[docs] section on this to get a thorough understanding on it. + +Here is it in a gist: + +Kafka supports two types of offsets to start with by default - `earliest` and `latest`. +Their semantics are self-explanatory from their names. + +Assuming you are running the consumer for the first time. +If you miss the group.id in your Spring Cloud Stream application, then it becomes an anonymous consumer. +Whenever, you have an anonymous consumer, in that case, Spring Cloud Stream application by default will start from the `latest` available offset in the topic partition. +On the other hand, if you explicitly specify a group.id, then by default, the Spring Cloud Stream application will start from the `earliest` available offset in the topic partiton. + +In both cases above (consumers with explicit groups and anonymous groups), the starting offset can be switched around by using the property `spring.cloud.stream.kafka.bindings..consumer.startOffset` and setting it to either `earliest` or `latest`. + +Now, assume that you already ran the consumer before and now starting it again. +In this case, the starting offset semantics in the above case do not apply as the consumer finds an already committed offset for the consumer group (In the case of an anonymous consumer, although the application does not provide a group.id, the binder will auto generate one for you). +It simply picks up from the last committed offset onward. +This is true, even when you have a `startOffset` value provided. + +However, you can override the default behavior where the consumer starts from the last committed offset by using the `resetOffsets` property. +In order to do that, set the property `spring.cloud.stream.kafka.bindings..consumer.resetOffsets` to `true` (which is `false` by default). +Then make sure you provide the `startOffset` value (either `earliest` or `latest`). +When you do that and then start the consumer application, each time you start, it starts as if this is starting for the first time and ignore any committed offsets for the partition. + +=== Seeking to arbitrary offsets in Kafka + +==== Problem Statement + +Using Kafka binder, I know that it can set the offset to either `earliest` or `latest`, but I have a requirement to seek the offset to something in the middle, an arbitrary offset. +Is there a way to achieve this using Spring Cloud Stream Kafka biner? + +==== Solution + +Previously we saw how Kafka binder allows you to tackle basic offset management. +By default, the binder does not allow you to rewind to an arbitrary offset, at least through the mechanism we saw in that reipce. +However, there are some low-level strategies that the binder provides to achieve this use case. +Let's explore them. + +First of all, when you want to reset to an arbitrary offset other than `earliest` or `latest`, make sure to leave the `resetOffsets` configuration to its defaults, which is `false`. +Then you have to provide a custom bean of type `KafkaBindingRebalanceListener`, which will be injected into all consumer bindings. +It is an interface that comes with a few default methods, but here is the method that we are interested in: + +``` +/** + * Invoked when partitions are initially assigned or after a rebalance. Applications + * might only want to perform seek operations on an initial assignment. While the + * 'initial' argument is true for each thread (when concurrency is greater than 1), + * implementations should keep track of exactly which partitions have been sought. + * There is a race in that a rebalance could occur during startup and so a topic/ + * partition that has been sought on one thread may be re-assigned to another + * thread and you may not wish to re-seek it at that time. + * @param bindingName the name of the binding. + * @param consumer the consumer. + * @param partitions the partitions. + * @param initial true if this is the initial assignment on the current thread. + */ + default void onPartitionsAssigned(String bindingName, Consumer consumer, + Collection partitions, boolean initial) { + // do nothing + } +``` + +Let us look at the details. + +In essence, this method will be invoked each time during the initial assignment for a topic partition or after a rebalance. +For better illustration, let us assume that our topic is `foo` and it has 4 partitions. +Initially, we are only starting a single consumer in the group and this consumer will consume from all partitions. +When the consumer starts for the first time, all 4 partitions are getting initially assigned. +However, we do not want to start the partitions to consume at the defaults (`earliest` since we define a group), rather for each partition, we want them to consume after seeking to arbitrary offsets. +Imagine that you have a business case to consume from certain offsets as below. + +``` +Partition start offset + +0 1000 +1 2000 +2 2000 +3 1000 +``` + +This could be achieved by implementing the above method as below. + +``` + +@Override +public void onPartitionsAssigned(String bindingName, Consumer consumer, Collection partitions, boolean initial) { + + Map topicPartitionOffset = new HashMap<>(); + topicPartitionOffset.put(new TopicPartition("foo", 0), 1000L); + topicPartitionOffset.put(new TopicPartition("foo", 1), 2000L); + topicPartitionOffset.put(new TopicPartition("foo", 2), 2000L); + topicPartitionOffset.put(new TopicPartition("foo", 3), 1000L); + + if (initial) { + partitions.forEach(tp -> { + if (topicPartitionOffset.containsKey(tp)) { + final Long offset = topicPartitionOffset.get(tp); + try { + consumer.seek(tp, offset); + } + catch (Exception e) { + // Handle excpetions carefully. + } + } + }); + } +} +``` + +This is just a rudimentary implementation. +Real world use cases are much more complex than this and you need to adjust accordingly, but this certainly gives you a basic sketch. +When consumer `seek` fails, it may throw some runtime exceptions and you need to decide what to do in those cases. + +==== What if we start a second consumer with the same group id? + +When we add a second consumer, a rebalance will occur and some partitions will be moved around. +Let's say that the new consumer gets partitions `2` and `3`. +When this new Spring Cloud Stream consumer calls this `onPartitionsAssigned` method, it will see that this is the initial assignment for partititon `2` and `3` on this consumer. +Therefore, it will do the seek operation becuase of the conditional check on the `initial` argument. +In the case of the first consumer, it now only has partitons `0` and `1` +However, for this consumer it was simply a rebalance event and not considered as an intial assignment. +Thus, it will not re-seek to the given offsets because of the conditional check on the `initial` argument. + +=== How do I manually acknowledge using Kafka binder? + +==== Problem Statement + +Using Kafka binder, I want to manually acknowledge messages in my consumer. +How do I do that? + +==== Solution + +By default, Kafka binder delegates to the default commit settings in Spring for Apache Kafka project. +The default `ackMode` in Spring Kafka is `batch`. +See https://docs.spring.io/spring-kafka/docs/current/reference/html/#committing-offsets[here] for more details on that. + +There are situations in which you want to disable this default commit behavior and rely on manual commits. +Following steps allow you to do that. + +Set the property `spring.cloud.stream.kafka.bindings..consumer.ackMode` to either `MANUAL` or `MANUAL_IMMEDIATE`. +When it is set like that, then there will be a header called `kafka_acknowledgment` (from `KafkaHeaders.ACKNOWLEDGMENT`) present in the message received by the consumer method. + +For example, imagine this as your consumer method. + +``` +@Bean +public Consumer> myConsumer() { + return msg -> { + Acknowledgment acknowledgment = message.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT, Acknowledgment.class); + if (acknowledgment != null) { + System.out.println("Acknowledgment provided"); + acknowledgment.acknowledge(); + } + }; +} +``` + +Then you set the property `spring.cloud.stream.bindings.myConsumer-in-0.consumer.ackMode` to `MANUAL` or `MANUAL_IMMEDIATE`. + +=== How do I override the default binding names in Spring Cloud Stream? + +==== Problem Statement + +Spring Cloud Stream creates default bindings based on the function definition and signature, but how do I override these to more domain friendly names? + +==== Solution + +Assume that following is your function signature. + +``` +@Bean +public Function uppercase(){ +... +} +``` + +By default, Spring Cloud Stream will create the bindings as below. + +1. uppercase-in-0 +2. uppercase-out-0 + +You can override these bindings to something by using the following properties. + +``` +spring.cloud.stream.function.bindings.uppercase-in-0=my-transformer-in +spring.cloud.stream.function.bindings.uppercase-out-0=my-transformer-out +``` + +After this, all binding properties must be made on the new names, `my-transformer-in` and `my-transformer-out`. + +Here is another example with Kafka Streams and multiple inputs. + +``` +@Bean +public BiFunction, KTable, KStream> processOrder() { +... +} +``` + +By default, Spring Cloud Stream will create three different binding names for this function. + +1. processOrder-in-0 +2. processOrder-in-1 +3. processOrder-out-0 + +You have to use these binding names each time you want to set some configuration on these bindings. +You don't like that, and you want to use more domain-friendly and readable binding names, for example, something like. + +1. orders +2. accounts +3. enrichedOrders + +You can easily do that by simply setting these three properties + +1. spring.cloud.stream.function.bindings.processOrder-in-0=orders +2. spring.cloud.stream.function.bindings.processOrder-in-1=accounts +3. spring.cloud.stream.function.bindings.processOrder-out-0=enrichedOrders + +Once you do that, it overrides the default binding names and any properties that you want to set on them must be on these new binding names. + +=== How do I send a message key as part of my record? + +==== Problem Statement + +I need to send a key along with the payload of the record, is there a way to do that in Spring Cloud Stream? + +==== Solution + +It is often necessary that you want to send associative data structure like a map as the record with a key and value. +Spring Cloud Stream allows you to do that in a straightforward manner. +Following is a basic blueprint for doing this, but you may want to adapt it to your paricular use case. + +Here is sample producer method (aka `Supplier`). + +``` +@Bean +public Supplier> supplier() { + return () -> MessageBuilder.withPayload("foo").setHeader(KafkaHeaders.MESSAGE_KEY, "my-foo").build(); +} +``` + +This is a trivial function that sends a message with a `String` payload, but also with a key. +Note that we set the key as a message header using `KafkaHeaders.MESSAGE_KEY`. + +If you want to change the key from the default `kafka_messageKey`, then in the configuration, we need to specify this property: + +``` +spring.cloud.stream.kafka.bindings.supplier-out-0.producer.messageKeyExpression=headers['my-special-key'] +``` + +Please note that we use the binding name `supplier-out-0` since that is our function name, please update accordingly. + +Then, we use this new key when we produce the message. + +=== How do I use native serializer and deserializer instead of message conversion done by Spring Cloud Stream? + +==== Problem Statement + +Instead of using the message converters in Spring Cloud Stream, I want to use native Serializer and Deserializer in Kafka. +By default, Spring Cloud Stream takes care of this conversion using its internal built-in message converters. +How can I bypass this and delegate the responsibility to Kafka? + +==== Solution + +This is really easy to do. + +All you have to do is to provide the following property to enable native serialization. + +``` +spring.cloud.stream.kafka.bindings..producer.useNativeEncoding: true +``` + +Then, you need to also set the serailzers. +There are a couple of ways to do this. + +``` +spring.cloud.stream.kafka.bindings..producer.configurarion.key.serializer: org.apache.kafka.common.serialization.StringSerializer +spring.cloud.stream.kafka.bindings..producer.configurarion.value.serializer: org.apache.kafka.common.serialization.StringSerializer +``` + +or using the binder configuration. + +``` +spring.cloud.stream.kafka.binder.configurarion.key.serializer: org.apache.kafka.common.serialization.StringSerializer +spring.cloud.stream.kafka.binder.configurarion.value.serializer: org.apache.kafka.common.serialization.StringSerializer +``` + +When using the binder way, it is applied against all the bindings whereas setting them at the bindings are per binding. + +On the deserializing side, you just need to provide the deserializers as configuration. + +For example, + +``` +spring.cloud.stream.kafka.bindings..consumer.configurarion.key.deserializer: org.apache.kafka.common.serialization.StringDeserializer +spring.cloud.stream.kafka.bindings..producer.configurarion.value.deserializer: org.apache.kafka.common.serialization.StringDeserializer +``` + +You can also set them at the binder level. + +There is an optional property that you can set to force native decoding. + +``` +spring.cloud.stream.kafka.bindings..consumer.useNativeDecoding: true +``` + +However, in the case of Kafka binder, this is unncessary, as by the time it reaches the binder, Kafka already deserializes them using the configured deserializers. + +=== Explain how offset resetting work in Kafka Streams binder + +==== Problem Statement + +By default, Kafka Streams binder always starts from the earliest offset for a new consumer. +Sometimes, it is beneficial or required by the application to start from the latest offset. +Kafka Streams binder allows you to do that. + +==== Solution + +Before we look at the solution, let us look at the following scenario. + +``` +@Bean +public BiConsumer, KTable> myBiConsumer{ + (s, t) -> s.join(t, ...) + ... +} +``` + +We have a `BiConsumer` bean that requires two input bindings. +In this case, the first binding is for a `KStream` and the second one is for a `KTable`. +When running this application for the first time, by default, both bindings start from the `earliest` offset. +What about I want to start from the `latest` offset due to some requirements? +You can do this by enabling the following properties. + +``` +spring.cloud.stream.kafka.streams.bindings.myBiConsumer-in-0.consumer.startOffset: latest +spring.cloud.stream.kafka.streams.bindings.myBiConsumer-in-1.consumer.startOffset: latest +``` + +If you want only one binding to start from the `latest` offset and the other to consumer from the default `earliest`, then leave the latter binding out from the configuration. + +Keep in mind that, once there are committed offsets, these setting are *not* honored and the committed offsets take precedence. + +=== Keeping track of successful sending of records (producing) in Kafka + +==== Problem Statement + +I have a Kafka producer application and I want to keep track of all my successful sedings. + +==== Solution + +Let us assume that we have this following supplier in the application. + +``` +@Bean + public Supplier> supplier() { + return () -> MessageBuilder.withPayload("foo").setHeader(KafkaHeaders.MESSAGE_KEY, "my-foo").build(); + } +``` + +Then, we need to define a new `MessageChannel` bean to capture all the successful send information. + +``` +@Bean + public MessageChannel fooRecordChannel() { + return new DirectChannel(); + } +``` + +Next, define this property in the application configuration to provide the bean name for the `recordMetadataChannel`. + +``` +spring.cloud.stream.kafka.bindings.supplier-out-0.producer.recordMetadataChannel: fooRecordChannel +``` + +At this point, successful sent information will be sent to the `fooRecordChannel`. + +You can write an `IntegrationFlow` as below to see the information. + +``` +@Bean +public IntegrationFlow integrationFlow() { + return f -> f.channel("fooRecordChannel") + .handle((payload, messageHeaders) -> payload); +} +``` + +In the `handle` method, the payload is what got sent to Kafka and the message headers contain a special key called `kafka_recordMetadata`. +Its value is a `RecordMetadata` that contains information about topic partition, current offset etc. + +=== Adding custom header mapper in Kafka + +==== Problem Statement + +I have a Kafka producer application that sets some headers, but they are missing in the consumer application. Why is that? + +==== Solution + +Under normal circumstances, this should be fine. + +Imagine, you have the following producer. + +``` +@Bean +public Supplier> supply() { + return () -> MessageBuilder.withPayload("foo").setHeader("foo", "bar").build(); +} +``` + +On the consumer side, you should still see the header "foo", and the following should not give you any issues. + +``` +@Bean +public Consumer> consume() { + return s -> { + final String foo = (String)s.getHeaders().get("foo"); + System.out.println(foo); + }; +} +``` + +If you provide a https://docs.spring.io/spring-cloud-stream-binder-kafka/docs/3.1.3/reference/html/spring-cloud-stream-binder-kafka.html#_kafka_binder_properties[custom header mapper] in the application, then this won't work. +Let's say you have an empty `KafkaHeaderMapper` in the application. + +``` +@Bean +public KafkaHeaderMapper kafkaBinderHeaderMapper() { + return new KafkaHeaderMapper() { + @Override + public void fromHeaders(MessageHeaders headers, Headers target) { + + } + + @Override + public void toHeaders(Headers source, Map target) { + + } + }; +} +``` + +If that is your implementation, then you will miss the `foo` header on the consumer. +Chances are that, you may have some logic inside those `KafkaHeaderMapper` methods. +You need the following to populate the `foo` header. + +``` +@Bean +public KafkaHeaderMapper kafkaBinderHeaderMapper() { + return new KafkaHeaderMapper() { + @Override + public void fromHeaders(MessageHeaders headers, Headers target) { + final String foo = (String) headers.get("foo"); + target.add("foo", foo.getBytes()); + } + + @Override + public void toHeaders(Headers source, Map target) { + final Header foo = source.lastHeader("foo"); + target.put("foo", new String(foo.value())); + } + } +``` + +That will properly populate the `foo` header from the producer to consumer. + +==== Special note on the id header + +In Spring Cloud Stream, the `id` header is a special header, but some applications may want to have special custom id headers - something like `custom-id` or `ID` or `Id`. +The first one (`custom-id`) will propagate without any custom header mapper from producer to consumer. +However, if you produce with a variant of the framework reserved `id` header - such as `ID`, `Id`, `iD` etc. then you will run into issues with the internals of the framework. +See this https://stackoverflow.com/questions/68412600/change-the-behaviour-in-spring-cloud-stream-make-header-matcher-case-sensitive[StackOverflow thread] fore more context on this use case. +In that case, you must use a custom `KafkaHeaderMapper` to map the case-sensitive id header. +For example, let's say you have the following producer. + +``` +@Bean +public Supplier> supply() { + return () -> MessageBuilder.withPayload("foo").setHeader("Id", "my-id").build(); +} +``` + +The header `Id` above will be gone from the consuming side as it clashes with the framework `id` header. +You can provide a custom `KafkaHeaderMapper` to solve this issue. + +``` +@Bean +public KafkaHeaderMapper kafkaBinderHeaderMapper1() { + return new KafkaHeaderMapper() { + @Override + public void fromHeaders(MessageHeaders headers, Headers target) { + final String myId = (String) headers.get("Id"); + target.add("Id", myId.getBytes()); + } + + @Override + public void toHeaders(Headers source, Map target) { + final Header Id = source.lastHeader("Id"); + target.put("Id", new String(Id.value())); + } + }; +} +``` + +By doing this, both `id` and `Id` headers will be available from the producer to the consumer side. + +=== Producing to multiple topics in transaction + +==== Problem Statement + +How do I produce transactional messages to multiple Kafka topics? + +For more context, see this https://stackoverflow.com/questions/68928091/dlq-bounded-retry-and-eos-when-producing-to-multiple-topics-using-spring-cloud[StackOverflow question]. + +==== Solution + +Use transactional support in Kafka binder for transactions and then provide an `AfterRollbackProcessor`. +In order to produce to multiple topics, use `StreamBridge` API. + +Below are the code snippets for this: + +``` +@Autowired +StreamBridge bridge; + +@Bean +Consumer input() { + return str -> { + System.out.println(str); + this.bridge.send("left", str.toUpperCase()); + this.bridge.send("right", str.toLowerCase()); + if (str.equals("Fail")) { + throw new RuntimeException("test"); + } + }; +} + +@Bean +ListenerContainerCustomizer> customizer(BinderFactory binders) { + return (container, dest, group) -> { + ProducerFactory pf = ((KafkaMessageChannelBinder) binders.getBinder(null, + MessageChannel.class)).getTransactionalProducerFactory(); + KafkaTemplate template = new KafkaTemplate<>(pf); + DefaultAfterRollbackProcessor rollbackProcessor = rollbackProcessor(template); + container.setAfterRollbackProcessor(rollbackProcessor); + }; +} + +DefaultAfterRollbackProcessor rollbackProcessor(KafkaTemplate template) { + return new DefaultAfterRollbackProcessor<>( + new DeadLetterPublishingRecoverer(template), new FixedBackOff(2000L, 2L), template, true); +} + +``` + +==== Required Configuration + +``` +spring.cloud.stream.kafka.binder.transaction.transaction-id-prefix: tx- +spring.cloud.stream.kafka.binder.required-acks=all +spring.cloud.stream.bindings.input-in-0.group=foo +spring.cloud.stream.bindings.input-in-0.destination=input +spring.cloud.stream.bindings.left.destination=left +spring.cloud.stream.bindings.right.destination=right + +spring.cloud.stream.kafka.bindings.input-in-0.consumer.maxAttempts=1 +``` + +in order to test, you can use the following: + +``` +@Bean +public ApplicationRunner runner(KafkaTemplate template) { + return args -> { + System.in.read(); + template.send("input", "Fail".getBytes()); + template.send("input", "Good".getBytes()); + }; +} +``` + +Some important notes: + +Please ensure that you don't have any DLQ settings on the application configuration as we manually configure DLT (By default it will be published to a topic named `input.DLT` based on the initial consumer function). +Also, reset the `maxAttempts` on consumer binding to `1` in order to avoid retries by the binder. +It will be max tried a total of 3 in the example above (initial try + the 2 attempts in the `FixedBackoff`). + +See the https://stackoverflow.com/questions/68928091/dlq-bounded-retry-and-eos-when-producing-to-multiple-topics-using-spring-cloud[StackOverflow thread] for more details on how to test this code. +If you are using Spring Cloud Stream to test it by adding more consumer functions, make sure to set the `isolation-level` on the consumer binding to `read-committed`. + +This https://stackoverflow.com/questions/68941306/spring-cloud-stream-database-transaction-does-not-roll-back[StackOverflow thread] is also related to this discussion. + +=== Pitfalls to avoid when running multiple pollable consumers + +==== Problem Statement + +How can I run multiple instances of the pollable consumers and generate unique `client.id` for each instance? + +==== Solution + +Assuming that I have the following definition: + +``` +spring.cloud.stream.pollable-source: foo +spring.cloud.stream.bindings.foo-in-0.group: my-group +``` + +When running the application, the Kafka consumer generates a client.id (something like `consumer-my-group-1`). +For each instance of the application that is running, this `client.id` will be the same, causing unexpected issues. + +In order to fix this, you can add the following property on each instance of the application: + +``` +spring.cloud.stream.kafka.bindings.foo-in-0.consumer.configuration.client.id=${client.id} +``` + +See this https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1139[GitHub issue] for more details. + diff --git a/k-binder/docs/src/main/ruby/generate_readme.sh b/k-binder/docs/src/main/ruby/generate_readme.sh new file mode 100755 index 000000000..6d0ce9dc5 --- /dev/null +++ b/k-binder/docs/src/main/ruby/generate_readme.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env ruby + +base_dir = File.join(File.dirname(__FILE__),'../../..') +src_dir = File.join(base_dir, "/src/main/asciidoc") +require 'asciidoctor' +require 'optparse' + +options = {} +file = "#{src_dir}/README.adoc" + +OptionParser.new do |o| + o.on('-o OUTPUT_FILE', 'Output file (default is stdout)') { |file| options[:to_file] = file unless file=='-' } + o.on('-h', '--help') { puts o; exit } + o.parse! +end + +file = ARGV[0] if ARGV.length>0 + +# Copied from https://github.com/asciidoctor/asciidoctor-extensions-lab/blob/master/scripts/asciidoc-coalescer.rb +doc = Asciidoctor.load_file file, safe: :unsafe, header_only: true, attributes: options[:attributes] +header_attr_names = (doc.instance_variable_get :@attributes_modified).to_a +header_attr_names.each {|k| doc.attributes[%(#{k}!)] = '' unless doc.attr? k } +attrs = doc.attributes +attrs['allow-uri-read'] = true +puts attrs + +out = "// Do not edit this file (e.g. go instead to src/main/asciidoc)\n\n" +doc = Asciidoctor.load_file file, safe: :unsafe, parse: false, attributes: attrs +out << doc.reader.read + +unless options[:to_file] + puts out +else + File.open(options[:to_file],'w+') do |file| + file.write(out) + end +end diff --git a/k-binder/mvnw b/k-binder/mvnw new file mode 100755 index 000000000..41c0f0c23 --- /dev/null +++ b/k-binder/mvnw @@ -0,0 +1,310 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + if [ -n "$MVNW_REPOURL" ]; then + jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + else + jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + fi + while IFS="=" read key value; do + case "$key" in (wrapperUrl) jarUrl="$value"; break ;; + esac + done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + if $cygwin; then + wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` + fi + + if command -v wget > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget "$jarUrl" -O "$wrapperJarPath" + else + wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl -o "$wrapperJarPath" "$jarUrl" -f + else + curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f + fi + + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaClass=`cygpath --path --windows "$javaClass"` + fi + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/k-binder/mvnw.cmd b/k-binder/mvnw.cmd new file mode 100644 index 000000000..86115719e --- /dev/null +++ b/k-binder/mvnw.cmd @@ -0,0 +1,182 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + +FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/k-binder/pom.xml b/k-binder/pom.xml new file mode 100644 index 000000000..da4ff50e9 --- /dev/null +++ b/k-binder/pom.xml @@ -0,0 +1,279 @@ + + + 4.0.0 + spring-cloud-stream-binder-kafka-parent + 3.2.3-SNAPSHOT + pom + + org.springframework.cloud + spring-cloud-build + 3.1.1 + + + + https://github.com/spring-cloud/spring-cloud-stream-binder-kafka + scm:git:git://github.com/spring-cloud/spring-cloud-stream-binder-kafka.git + + + scm:git:ssh://git@github.com/spring-cloud/spring-cloud-stream-binder-kafka.git + + HEAD + + + 1.8 + 2.8.4-SNAPSHOT + 5.5.8 + 3.0.0 + 3.2.3-SNAPSHOT + true + true + true + + + spring-cloud-stream-binder-kafka + spring-cloud-starter-stream-kafka + spring-cloud-stream-binder-kafka-core + spring-cloud-stream-binder-kafka-streams + docs + + + + + + org.springframework.cloud + spring-cloud-stream-binder-kafka-core + ${project.version} + + + org.springframework.cloud + spring-cloud-stream-binder-kafka + ${project.version} + + + org.springframework.cloud + spring-cloud-stream + ${spring-cloud-stream.version} + + + org.apache.kafka + kafka-clients + ${kafka.version} + + + + org.springframework.kafka + spring-kafka + ${spring-kafka.version} + + + org.springframework.integration + spring-integration-kafka + ${spring-integration-kafka.version} + + + org.springframework.cloud + spring-cloud-stream-binder-test + ${spring-cloud-stream.version} + test + + + org.springframework.kafka + spring-kafka-test + test + ${spring-kafka.version} + + + org.apache.kafka + kafka-streams + ${kafka.version} + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.kafka + kafka_2.13 + ${kafka.version} + test + + + org.apache.kafka + kafka_2.13 + test + test + ${kafka.version} + + + jline + jline + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + + org.apache.kafka + kafka-clients + ${kafka.version} + test + test + + + + + + + org.junit.vintage + junit-vintage-engine + test + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.7 + + + org.apache.maven.plugins + maven-javadoc-plugin + + true + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + ${java.version} + ${java.version} + -parameters + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + + + + + spring + + + + + coverage + + + env.TRAVIS + true + + + + + + org.jacoco + jacoco-maven-plugin + 0.7.9 + + + agent + + prepare-agent + + + + report + test + + report + + + + + + + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + spring-milestones + Spring milestones + https://repo.spring.io/libs-milestone-local + + + rsocket-snapshots + RSocket Snapshots + https://oss.jfrog.org/oss-snapshot-local + + true + + + + spring-releases + Spring Releases + https://repo.spring.io/release + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/snapshot + + true + + + + spring-milestones + Spring Milestones + https://repo.spring.io/milestone + + false + + + + spring-releases + Spring Releases + https://repo.spring.io/release + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + + diff --git a/k-binder/spring-cloud-starter-stream-kafka/.jdk8 b/k-binder/spring-cloud-starter-stream-kafka/.jdk8 new file mode 100644 index 000000000..e69de29bb diff --git a/k-binder/spring-cloud-starter-stream-kafka/pom.xml b/k-binder/spring-cloud-starter-stream-kafka/pom.xml new file mode 100644 index 000000000..1bb40f592 --- /dev/null +++ b/k-binder/spring-cloud-starter-stream-kafka/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + org.springframework.cloud + spring-cloud-stream-binder-kafka-parent + 3.2.3-SNAPSHOT + + spring-cloud-starter-stream-kafka + Spring Cloud Starter Stream Kafka + https://projects.spring.io/spring-cloud + + Pivotal Software, Inc. + https://www.spring.io + + + ${basedir}/../.. + + + + org.springframework.cloud + spring-cloud-stream-binder-kafka + ${project.version} + + + diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/.jdk8 b/k-binder/spring-cloud-stream-binder-kafka-core/.jdk8 new file mode 100644 index 000000000..e69de29bb diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/.settings/org.eclipse.jdt.ui.prefs b/k-binder/spring-cloud-stream-binder-kafka-core/.settings/org.eclipse.jdt.ui.prefs new file mode 100644 index 000000000..f9aac64a9 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/.settings/org.eclipse.jdt.ui.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.ui.ignorelowercasenames=true +org.eclipse.jdt.ui.importorder=java;javax;com;org;org.springframework;ch.qos;\#; +org.eclipse.jdt.ui.ondemandthreshold=99 +org.eclipse.jdt.ui.staticondemandthreshold=99 diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/pom.xml b/k-binder/spring-cloud-stream-binder-kafka-core/pom.xml new file mode 100644 index 000000000..1e8c4eb07 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/pom.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + + org.springframework.cloud + spring-cloud-stream-binder-kafka-parent + 3.2.3-SNAPSHOT + + spring-cloud-stream-binder-kafka-core + Spring Cloud Stream Kafka Binder Core + https://projects.spring.io/spring-cloud + + Pivotal Software, Inc. + https://www.spring.io + + + + + org.springframework.cloud + spring-cloud-stream + + + org.springframework.integration + spring-integration-kafka + + + org.springframework.boot + spring-boot-configuration-processor + true + + + org.springframework.boot + spring-boot-test + test + + + org.springframework.kafka + spring-kafka-test + test + + + org.springframework.cloud + spring-cloud-stream-binder-test + test + + + + diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/JaasLoginModuleConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/JaasLoginModuleConfiguration.java new file mode 100644 index 000000000..9934f30dd --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/JaasLoginModuleConfiguration.java @@ -0,0 +1,69 @@ +/* + * Copyright 2016-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.properties; + +import java.util.HashMap; +import java.util.Map; + +import javax.security.auth.login.AppConfigurationEntry; + +import org.springframework.kafka.security.jaas.KafkaJaasLoginModuleInitializer; +import org.springframework.util.Assert; + +/** + * Contains properties for setting up an {@link AppConfigurationEntry} that can be used + * for the Kafka or Zookeeper client. + * + * @author Marius Bogoevici + * @author Soby Chacko + */ +public class JaasLoginModuleConfiguration { + + private String loginModule = "com.sun.security.auth.module.Krb5LoginModule"; + + private KafkaJaasLoginModuleInitializer.ControlFlag controlFlag = KafkaJaasLoginModuleInitializer.ControlFlag.REQUIRED; + + private Map options = new HashMap<>(); + + public String getLoginModule() { + return this.loginModule; + } + + public void setLoginModule(String loginModule) { + Assert.notNull(loginModule, "cannot be null"); + this.loginModule = loginModule; + } + + public KafkaJaasLoginModuleInitializer.ControlFlag getControlFlag() { + return this.controlFlag; + } + + public void setControlFlag(String controlFlag) { + Assert.notNull(controlFlag, "cannot be null"); + this.controlFlag = KafkaJaasLoginModuleInitializer.ControlFlag + .valueOf(controlFlag.toUpperCase()); + } + + public Map getOptions() { + return this.options; + } + + public void setOptions(Map options) { + this.options = options; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java new file mode 100644 index 000000000..c53566eef --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java @@ -0,0 +1,669 @@ +/* + * Copyright 2015-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.properties; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import javax.validation.constraints.AssertTrue; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; + +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.cloud.stream.binder.HeaderMode; +import org.springframework.cloud.stream.binder.ProducerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties.CompressionType; +import org.springframework.core.io.DefaultResourceLoader; +import org.springframework.core.io.Resource; +import org.springframework.expression.Expression; +import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; +import org.springframework.util.StringUtils; + +/** + * Configuration properties for the Kafka binder. The properties in this class are + * prefixed with spring.cloud.stream.kafka.binder. + * + * @author David Turanski + * @author Ilayaperumal Gopinathan + * @author Marius Bogoevici + * @author Soby Chacko + * @author Gary Russell + * @author Rafal Zukowski + * @author Aldo Sinanaj + * @author Lukasz Kaminski + * @author Chukwubuikem Ume-Ugwa + */ +@ConfigurationProperties(prefix = "spring.cloud.stream.kafka.binder") +public class KafkaBinderConfigurationProperties { + + private static final String DEFAULT_KAFKA_CONNECTION_STRING = "localhost:9092"; + + private final Log logger = LogFactory.getLog(getClass()); + + private final Transaction transaction = new Transaction(); + + private final KafkaProperties kafkaProperties; + + /** + * Arbitrary kafka properties that apply to both producers and consumers. + */ + private Map configuration = new HashMap<>(); + + /** + * Arbitrary kafka consumer properties. + */ + private Map consumerProperties = new HashMap<>(); + + /** + * Arbitrary kafka producer properties. + */ + private Map producerProperties = new HashMap<>(); + + private String[] brokers = new String[] { "localhost" }; + + private String defaultBrokerPort = "9092"; + + private String[] headers = new String[] {}; + + private boolean autoCreateTopics = true; + + private boolean autoAlterTopics; + + private boolean autoAddPartitions; + + private boolean considerDownWhenAnyPartitionHasNoLeader; + + private String requiredAcks = "1"; + + private short replicationFactor = -1; + + private int minPartitionCount = 1; + + /** + * Time to wait to get partition information in seconds; default 60. + */ + private int healthTimeout = 60; + + private JaasLoginModuleConfiguration jaas; + + /** + * The bean name of a custom header mapper to use instead of a + * {@link org.springframework.kafka.support.DefaultKafkaHeaderMapper}. + */ + private String headerMapperBeanName; + + /** + * Time between retries after AuthorizationException is caught in + * the ListenerContainer; defalt is null which disables retries. + * For more info see: {@link org.springframework.kafka.listener.ConsumerProperties#setAuthorizationExceptionRetryInterval(java.time.Duration)} + */ + private Duration authorizationExceptionRetryInterval; + + /** + * When a certificate store location is given as classpath URL (classpath:), then the binder + * moves the resource from the classpath location inside the JAR to a location on + * the filesystem. If this value is set, then this location is used, otherwise, the + * certificate file is copied to the directory returned by java.io.tmpdir. + */ + private String certificateStoreDirectory; + + public KafkaBinderConfigurationProperties(KafkaProperties kafkaProperties) { + Assert.notNull(kafkaProperties, "'kafkaProperties' cannot be null"); + this.kafkaProperties = kafkaProperties; + } + + public KafkaProperties getKafkaProperties() { + return this.kafkaProperties; + } + + public Transaction getTransaction() { + return this.transaction; + } + + public String getKafkaConnectionString() { + // We need to do a check on certificate file locations to see if they are given as classpath resources. + // If that is the case, then we will move them to a file system location and use those as the certificate locations. + // This is due to a limitation in Kafka itself in which it doesn't allow reading certificate resources from the classpath. + // See this: https://issues.apache.org/jira/browse/KAFKA-7685 + // and this: https://cwiki.apache.org/confluence/display/KAFKA/KIP-398%3A+Support+reading+trust+store+from+classpath + moveCertsToFileSystemIfNecessary(); + + return toConnectionString(this.brokers, this.defaultBrokerPort); + } + + private void moveCertsToFileSystemIfNecessary() { + try { + moveBrokerCertsIfApplicable(); + moveSchemaRegistryCertsIfApplicable(); + } + catch (Exception e) { + throw new IllegalStateException(e); + } + } + + private void moveBrokerCertsIfApplicable() throws IOException { + final String trustStoreLocation = this.configuration.get("ssl.truststore.location"); + if (trustStoreLocation != null && trustStoreLocation.startsWith("classpath:")) { + final String fileSystemLocation = moveCertToFileSystem(trustStoreLocation, this.certificateStoreDirectory); + // Overriding the value with absolute filesystem path. + this.configuration.put("ssl.truststore.location", fileSystemLocation); + } + final String keyStoreLocation = this.configuration.get("ssl.keystore.location"); + if (keyStoreLocation != null && keyStoreLocation.startsWith("classpath:")) { + final String fileSystemLocation = moveCertToFileSystem(keyStoreLocation, this.certificateStoreDirectory); + // Overriding the value with absolute filesystem path. + this.configuration.put("ssl.keystore.location", fileSystemLocation); + } + } + + private void moveSchemaRegistryCertsIfApplicable() throws IOException { + String trustStoreLocation = this.configuration.get("schema.registry.ssl.truststore.location"); + if (trustStoreLocation != null && trustStoreLocation.startsWith("classpath:")) { + final String fileSystemLocation = moveCertToFileSystem(trustStoreLocation, this.certificateStoreDirectory); + // Overriding the value with absolute filesystem path. + this.configuration.put("schema.registry.ssl.truststore.location", fileSystemLocation); + } + final String keyStoreLocation = this.configuration.get("schema.registry.ssl.keystore.location"); + if (keyStoreLocation != null && keyStoreLocation.startsWith("classpath:")) { + final String fileSystemLocation = moveCertToFileSystem(keyStoreLocation, this.certificateStoreDirectory); + // Overriding the value with absolute filesystem path. + this.configuration.put("schema.registry.ssl.keystore.location", fileSystemLocation); + } + } + + private String moveCertToFileSystem(String classpathLocation, String fileSystemLocation) throws IOException { + File targetFile; + final String tempDir = System.getProperty("java.io.tmpdir"); + Resource resource = new DefaultResourceLoader().getResource(classpathLocation); + if (StringUtils.hasText(fileSystemLocation)) { + final Path path = Paths.get(fileSystemLocation); + if (!Files.exists(path) || !Files.isDirectory(path) || !Files.isWritable(path)) { + logger.warn("The filesystem location to move the cert files (" + fileSystemLocation + ") " + + "is not found or a directory that is writable. The system temp folder (java.io.tmpdir) will be used instead."); + targetFile = new File(Paths.get(tempDir, resource.getFilename()).toString()); + } + else { + // the given location is verified to be a writable directory. + targetFile = new File(Paths.get(fileSystemLocation, resource.getFilename()).toString()); + } + } + else { + targetFile = new File(Paths.get(tempDir, resource.getFilename()).toString()); + } + + try (InputStream inputStream = resource.getInputStream()) { + Files.copy(inputStream, targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + return targetFile.getAbsolutePath(); + } + + public String getDefaultKafkaConnectionString() { + return DEFAULT_KAFKA_CONNECTION_STRING; + } + + public String[] getHeaders() { + return this.headers; + } + + public String[] getBrokers() { + return this.brokers; + } + + public void setBrokers(String... brokers) { + this.brokers = brokers; + } + + public void setDefaultBrokerPort(String defaultBrokerPort) { + this.defaultBrokerPort = defaultBrokerPort; + } + + public void setHeaders(String... headers) { + this.headers = headers; + } + + /** + * Converts an array of host values to a comma-separated String. It will append the + * default port value, if not already specified. + * @param hosts host string + * @param defaultPort port + * @return formatted connection string + */ + private String toConnectionString(String[] hosts, String defaultPort) { + String[] fullyFormattedHosts = new String[hosts.length]; + for (int i = 0; i < hosts.length; i++) { + if (hosts[i].contains(":") || StringUtils.isEmpty(defaultPort)) { + fullyFormattedHosts[i] = hosts[i]; + } + else { + fullyFormattedHosts[i] = hosts[i] + ":" + defaultPort; + } + } + return StringUtils.arrayToCommaDelimitedString(fullyFormattedHosts); + } + + public String getRequiredAcks() { + return this.requiredAcks; + } + + public void setRequiredAcks(String requiredAcks) { + this.requiredAcks = requiredAcks; + } + + public short getReplicationFactor() { + return this.replicationFactor; + } + + public void setReplicationFactor(short replicationFactor) { + this.replicationFactor = replicationFactor; + } + + public int getMinPartitionCount() { + return this.minPartitionCount; + } + + public void setMinPartitionCount(int minPartitionCount) { + this.minPartitionCount = minPartitionCount; + } + + public int getHealthTimeout() { + return this.healthTimeout; + } + + public void setHealthTimeout(int healthTimeout) { + this.healthTimeout = healthTimeout; + } + + public boolean isAutoCreateTopics() { + return this.autoCreateTopics; + } + + public void setAutoCreateTopics(boolean autoCreateTopics) { + this.autoCreateTopics = autoCreateTopics; + } + + public boolean isAutoAlterTopics() { + return autoAlterTopics; + } + + public void setAutoAlterTopics(boolean autoAlterTopics) { + this.autoAlterTopics = autoAlterTopics; + } + + public boolean isAutoAddPartitions() { + return this.autoAddPartitions; + } + + public void setAutoAddPartitions(boolean autoAddPartitions) { + this.autoAddPartitions = autoAddPartitions; + } + + public Map getConfiguration() { + return this.configuration; + } + + public void setConfiguration(Map configuration) { + this.configuration = configuration; + } + + public Map getConsumerProperties() { + return this.consumerProperties; + } + + public void setConsumerProperties(Map consumerProperties) { + Assert.notNull(consumerProperties, "'consumerProperties' cannot be null"); + this.consumerProperties = consumerProperties; + } + + public Map getProducerProperties() { + return this.producerProperties; + } + + public void setProducerProperties(Map producerProperties) { + Assert.notNull(producerProperties, "'producerProperties' cannot be null"); + this.producerProperties = producerProperties; + } + + /** + * Merge boot consumer properties, general properties from + * {@link #setConfiguration(Map)} that apply to consumers, properties from + * {@link #setConsumerProperties(Map)}, in that order. + * @return the merged properties. + */ + public Map mergedConsumerConfiguration() { + Map consumerConfiguration = new HashMap<>(); + consumerConfiguration.putAll(this.kafkaProperties.buildConsumerProperties()); + // Copy configured binder properties that apply to consumers + for (Map.Entry configurationEntry : this.configuration + .entrySet()) { + if (ConsumerConfig.configNames().contains(configurationEntry.getKey())) { + consumerConfiguration.put(configurationEntry.getKey(), + configurationEntry.getValue()); + } + } + consumerConfiguration.putAll(this.consumerProperties); + filterStreamManagedConfiguration(consumerConfiguration); + // Override Spring Boot bootstrap server setting if left to default with the value + // configured in the binder + return getConfigurationWithBootstrapServer(consumerConfiguration, + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG); + } + + /** + * Merge boot producer properties, general properties from + * {@link #setConfiguration(Map)} that apply to producers, properties from + * {@link #setProducerProperties(Map)}, in that order. + * @return the merged properties. + */ + public Map mergedProducerConfiguration() { + Map producerConfiguration = new HashMap<>(); + producerConfiguration.putAll(this.kafkaProperties.buildProducerProperties()); + // Copy configured binder properties that apply to producers + for (Map.Entry configurationEntry : this.configuration + .entrySet()) { + if (ProducerConfig.configNames().contains(configurationEntry.getKey())) { + producerConfiguration.put(configurationEntry.getKey(), + configurationEntry.getValue()); + } + } + producerConfiguration.putAll(this.producerProperties); + // Override Spring Boot bootstrap server setting if left to default with the value + // configured in the binder + return getConfigurationWithBootstrapServer(producerConfiguration, + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG); + } + + private void filterStreamManagedConfiguration(Map configuration) { + if (configuration.containsKey(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG) + && configuration.get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG).equals(true)) { + logger.warn(constructIgnoredConfigMessage(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG) + + ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG + "=true is not supported by the Kafka binder"); + configuration.remove(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG); + } + if (configuration.containsKey(ConsumerConfig.GROUP_ID_CONFIG)) { + logger.warn(constructIgnoredConfigMessage(ConsumerConfig.GROUP_ID_CONFIG) + + "Use spring.cloud.stream.default.group or spring.cloud.stream.binding..group to specify " + + "the group instead of " + ConsumerConfig.GROUP_ID_CONFIG); + configuration.remove(ConsumerConfig.GROUP_ID_CONFIG); + } + } + + private String constructIgnoredConfigMessage(String config) { + return String.format("Ignoring provided value(s) for '%s'. ", config); + } + + private Map getConfigurationWithBootstrapServer( + Map configuration, String bootstrapServersConfig) { + final String kafkaConnectionString = getKafkaConnectionString(); + if (ObjectUtils.isEmpty(configuration.get(bootstrapServersConfig)) || + !kafkaConnectionString.equals("localhost:9092")) { + configuration.put(bootstrapServersConfig, kafkaConnectionString); + } + return Collections.unmodifiableMap(configuration); + } + + public JaasLoginModuleConfiguration getJaas() { + return this.jaas; + } + + public void setJaas(JaasLoginModuleConfiguration jaas) { + this.jaas = jaas; + } + + public String getHeaderMapperBeanName() { + return this.headerMapperBeanName; + } + + public void setHeaderMapperBeanName(String headerMapperBeanName) { + this.headerMapperBeanName = headerMapperBeanName; + } + + public Duration getAuthorizationExceptionRetryInterval() { + return authorizationExceptionRetryInterval; + } + + public void setAuthorizationExceptionRetryInterval(Duration authorizationExceptionRetryInterval) { + this.authorizationExceptionRetryInterval = authorizationExceptionRetryInterval; + } + + public boolean isConsiderDownWhenAnyPartitionHasNoLeader() { + return this.considerDownWhenAnyPartitionHasNoLeader; + } + + public void setConsiderDownWhenAnyPartitionHasNoLeader(boolean considerDownWhenAnyPartitionHasNoLeader) { + this.considerDownWhenAnyPartitionHasNoLeader = considerDownWhenAnyPartitionHasNoLeader; + } + + public String getCertificateStoreDirectory() { + return this.certificateStoreDirectory; + } + + public void setCertificateStoreDirectory(String certificateStoreDirectory) { + this.certificateStoreDirectory = certificateStoreDirectory; + } + + /** + * Domain class that models transaction capabilities in Kafka. + */ + public static class Transaction { + + private final CombinedProducerProperties producer = new CombinedProducerProperties(); + + private String transactionIdPrefix; + + public String getTransactionIdPrefix() { + return this.transactionIdPrefix; + } + + public void setTransactionIdPrefix(String transactionIdPrefix) { + this.transactionIdPrefix = transactionIdPrefix; + } + + public CombinedProducerProperties getProducer() { + return this.producer; + } + + } + + /** + * An combination of {@link ProducerProperties} and {@link KafkaProducerProperties} so + * that common and kafka-specific properties can be set for the transactional + * producer. + * + * @since 2.1 + */ + public static class CombinedProducerProperties { + + private final ProducerProperties producerProperties = new ProducerProperties(); + + private final KafkaProducerProperties kafkaProducerProperties = new KafkaProducerProperties(); + + public Expression getPartitionKeyExpression() { + return this.producerProperties.getPartitionKeyExpression(); + } + + public void setPartitionKeyExpression(Expression partitionKeyExpression) { + this.producerProperties.setPartitionKeyExpression(partitionKeyExpression); + } + + public boolean isPartitioned() { + return this.producerProperties.isPartitioned(); + } + + public Expression getPartitionSelectorExpression() { + return this.producerProperties.getPartitionSelectorExpression(); + } + + public void setPartitionSelectorExpression( + Expression partitionSelectorExpression) { + this.producerProperties + .setPartitionSelectorExpression(partitionSelectorExpression); + } + + public @Min(value = 1, message = "Partition count should be greater than zero.") int getPartitionCount() { + return this.producerProperties.getPartitionCount(); + } + + public void setPartitionCount(int partitionCount) { + this.producerProperties.setPartitionCount(partitionCount); + } + + public String[] getRequiredGroups() { + return this.producerProperties.getRequiredGroups(); + } + + public void setRequiredGroups(String... requiredGroups) { + this.producerProperties.setRequiredGroups(requiredGroups); + } + + public @AssertTrue(message = "Partition key expression and partition key extractor class properties " + + "are mutually exclusive.") boolean isValidPartitionKeyProperty() { + return this.producerProperties.isValidPartitionKeyProperty(); + } + + public @AssertTrue(message = "Partition selector class and partition selector expression " + + "properties are mutually exclusive.") boolean isValidPartitionSelectorProperty() { + return this.producerProperties.isValidPartitionSelectorProperty(); + } + + public HeaderMode getHeaderMode() { + return this.producerProperties.getHeaderMode(); + } + + public void setHeaderMode(HeaderMode headerMode) { + this.producerProperties.setHeaderMode(headerMode); + } + + public boolean isUseNativeEncoding() { + return this.producerProperties.isUseNativeEncoding(); + } + + public void setUseNativeEncoding(boolean useNativeEncoding) { + this.producerProperties.setUseNativeEncoding(useNativeEncoding); + } + + public boolean isErrorChannelEnabled() { + return this.producerProperties.isErrorChannelEnabled(); + } + + public void setErrorChannelEnabled(boolean errorChannelEnabled) { + this.producerProperties.setErrorChannelEnabled(errorChannelEnabled); + } + + public String getPartitionKeyExtractorName() { + return this.producerProperties.getPartitionKeyExtractorName(); + } + + public void setPartitionKeyExtractorName(String partitionKeyExtractorName) { + this.producerProperties + .setPartitionKeyExtractorName(partitionKeyExtractorName); + } + + public String getPartitionSelectorName() { + return this.producerProperties.getPartitionSelectorName(); + } + + public void setPartitionSelectorName(String partitionSelectorName) { + this.producerProperties.setPartitionSelectorName(partitionSelectorName); + } + + public int getBufferSize() { + return this.kafkaProducerProperties.getBufferSize(); + } + + public void setBufferSize(int bufferSize) { + this.kafkaProducerProperties.setBufferSize(bufferSize); + } + + public @NotNull CompressionType getCompressionType() { + return this.kafkaProducerProperties.getCompressionType(); + } + + public void setCompressionType(CompressionType compressionType) { + this.kafkaProducerProperties.setCompressionType(compressionType); + } + + public boolean isSync() { + return this.kafkaProducerProperties.isSync(); + } + + public void setSync(boolean sync) { + this.kafkaProducerProperties.setSync(sync); + } + + public int getBatchTimeout() { + return this.kafkaProducerProperties.getBatchTimeout(); + } + + public void setBatchTimeout(int batchTimeout) { + this.kafkaProducerProperties.setBatchTimeout(batchTimeout); + } + + public Expression getMessageKeyExpression() { + return this.kafkaProducerProperties.getMessageKeyExpression(); + } + + public void setMessageKeyExpression(Expression messageKeyExpression) { + this.kafkaProducerProperties.setMessageKeyExpression(messageKeyExpression); + } + + public String[] getHeaderPatterns() { + return this.kafkaProducerProperties.getHeaderPatterns(); + } + + public void setHeaderPatterns(String[] headerPatterns) { + this.kafkaProducerProperties.setHeaderPatterns(headerPatterns); + } + + public Map getConfiguration() { + return this.kafkaProducerProperties.getConfiguration(); + } + + public void setConfiguration(Map configuration) { + this.kafkaProducerProperties.setConfiguration(configuration); + } + + public KafkaTopicProperties getTopic() { + return this.kafkaProducerProperties.getTopic(); + } + + public void setTopic(KafkaTopicProperties topic) { + this.kafkaProducerProperties.setTopic(topic); + } + + public KafkaProducerProperties getExtension() { + return this.kafkaProducerProperties; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBindingProperties.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBindingProperties.java new file mode 100644 index 000000000..e84b0b2c7 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBindingProperties.java @@ -0,0 +1,63 @@ +/* + * Copyright 2016-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.properties; + +import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider; + +/** + * Container object for Kafka specific extended producer and consumer binding properties. + * + * @author Marius Bogoevici + * @author Oleg Zhurakousky + */ +public class KafkaBindingProperties implements BinderSpecificPropertiesProvider { + + /** + * Consumer specific binding properties. @see {@link KafkaConsumerProperties}. + */ + private KafkaConsumerProperties consumer = new KafkaConsumerProperties(); + + /** + * Producer specific binding properties. @see {@link KafkaProducerProperties}. + */ + private KafkaProducerProperties producer = new KafkaProducerProperties(); + + /** + * @return {@link KafkaConsumerProperties} + * Consumer specific binding properties. @see {@link KafkaConsumerProperties}. + */ + public KafkaConsumerProperties getConsumer() { + return this.consumer; + } + + public void setConsumer(KafkaConsumerProperties consumer) { + this.consumer = consumer; + } + + /** + * @return {@link KafkaProducerProperties} + * Producer specific binding properties. @see {@link KafkaProducerProperties}. + */ + public KafkaProducerProperties getProducer() { + return this.producer; + } + + public void setProducer(KafkaProducerProperties producer) { + this.producer = producer; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaConsumerProperties.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaConsumerProperties.java new file mode 100644 index 000000000..0a4d2561d --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaConsumerProperties.java @@ -0,0 +1,545 @@ +/* + * Copyright 2016-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.properties; + +import java.util.HashMap; +import java.util.Map; + +import org.springframework.kafka.listener.ContainerProperties; + +/** + * Extended consumer properties for Kafka binder. + * + * @author Marius Bogoevici + * @author Ilayaperumal Gopinathan + * @author Soby Chacko + * @author Gary Russell + * @author Aldo Sinanaj + * + *

+ * Thanks to Laszlo Szabo for providing the initial patch for generic property support. + *

+ */ +public class KafkaConsumerProperties { + + /** + * Enumeration for starting consumer offset. + */ + public enum StartOffset { + + /** + * Starting from earliest offset. + */ + earliest(-2L), + /** + * Starting from latest offset. + */ + latest(-1L); + + private final long referencePoint; + + StartOffset(long referencePoint) { + this.referencePoint = referencePoint; + } + + public long getReferencePoint() { + return this.referencePoint; + } + + } + + /** + * Standard headers for the message. + */ + public enum StandardHeaders { + + /** + * No headers. + */ + none, + /** + * Message header representing ID. + */ + id, + /** + * Message header representing timestamp. + */ + timestamp, + /** + * Indicating both ID and timestamp headers. + */ + both + + } + + /** + * When true the offset is committed after each record, otherwise the offsets for the complete set of records + * received from the poll() are committed after all records have been processed. + */ + @Deprecated + private boolean ackEachRecord; + + /** + * When true, topic partitions is automatically rebalanced between the members of a consumer group. + * When false, each consumer is assigned a fixed set of partitions based on spring.cloud.stream.instanceCount and spring.cloud.stream.instanceIndex. + */ + private boolean autoRebalanceEnabled = true; + + /** + * Whether to autocommit offsets when a message has been processed. + * If set to false, a header with the key kafka_acknowledgment of the type org.springframework.kafka.support.Acknowledgment header + * is present in the inbound message. Applications may use this header for acknowledging messages. + */ + @Deprecated + private boolean autoCommitOffset = true; + + /** + * Controlling the container acknowledgement mode. This is the preferred way to control the ack mode on the + * container instead of the deprecated autoCommitOffset property. + */ + private ContainerProperties.AckMode ackMode; + + /** + * Flag to enable auto commit on error in polled consumers. + */ + private Boolean autoCommitOnError; + + /** + * The starting offset for new groups. Allowed values: earliest and latest. + */ + private StartOffset startOffset; + + /** + * Whether to reset offsets on the consumer to the value provided by startOffset. + * Must be false if a KafkaRebalanceListener is provided. + */ + private boolean resetOffsets; + + /** + * When set to true, it enables DLQ behavior for the consumer. + * By default, messages that result in errors are forwarded to a topic named error.name-of-destination.name-of-group. + * The DLQ topic name can be configurable by setting the dlqName property. + */ + private boolean enableDlq; + + /** + * The name of the DLQ topic to receive the error messages. + */ + private String dlqName; + + /** + * Number of partitions to use on the DLQ. + */ + private Integer dlqPartitions; + + /** + * Using this, DLQ-specific producer properties can be set. + * All the properties available through kafka producer properties can be set through this property. + */ + private KafkaProducerProperties dlqProducerProperties = new KafkaProducerProperties(); + + /** + * @deprecated No longer used by the binder. + */ + @Deprecated + private int recoveryInterval = 5000; + + /** + * List of trusted packages to provide the header mapper. + */ + private String[] trustedPackages; + + /** + * Indicates which standard headers are populated by the inbound channel adapter. + * Allowed values: none, id, timestamp, or both. + */ + private StandardHeaders standardHeaders = StandardHeaders.none; + + /** + * The name of a bean that implements RecordMessageConverter. + */ + private String converterBeanName; + + /** + * The interval, in milliseconds, between events indicating that no messages have recently been received. + */ + private long idleEventInterval = 30_000; + + /** + * When true, the destination is treated as a regular expression Pattern used to match topic names by the broker. + */ + private boolean destinationIsPattern; + + /** + * Map with a key/value pair containing generic Kafka consumer properties. + * In addition to having Kafka consumer properties, other configuration properties can be passed here. + */ + private Map configuration = new HashMap<>(); + + /** + * Various topic level properties. @see {@link KafkaTopicProperties} for more details. + */ + private KafkaTopicProperties topic = new KafkaTopicProperties(); + + /** + * Timeout used for polling in pollable consumers. + */ + private long pollTimeout = org.springframework.kafka.listener.ConsumerProperties.DEFAULT_POLL_TIMEOUT; + + /** + * Transaction manager bean name - overrides the binder's transaction configuration. + */ + private String transactionManager; + + /** + * Set to false to NOT commit the offset of a successfully recovered recovered in the after rollback processor. + */ + private boolean txCommitRecovered = true; + + /** + * CommonErrorHandler bean name per consumer binding. + * @since 3.2 + */ + private String commonErrorHandlerBeanName; + + /** + * @return if each record needs to be acknowledged. + * + * When true the offset is committed after each record, otherwise the offsets for the complete set of records + * received from the poll() are committed after all records have been processed. + * + * @deprecated since 3.1 in favor of using {@link #ackMode} + */ + @Deprecated + public boolean isAckEachRecord() { + return this.ackEachRecord; + } + + /** + * @param ackEachRecord + * + * @deprecated in favor of using {@link #ackMode} + */ + @Deprecated + public void setAckEachRecord(boolean ackEachRecord) { + this.ackEachRecord = ackEachRecord; + } + + /** + * @return is autocommit offset enabled + * + * Whether to autocommit offsets when a message has been processed. + * If set to false, a header with the key kafka_acknowledgment of the type org.springframework.kafka.support.Acknowledgment header + * is present in the inbound message. Applications may use this header for acknowledging messages. + * + * @deprecated since 3.1 in favor of using {@link #ackMode} + */ + @Deprecated + public boolean isAutoCommitOffset() { + return this.autoCommitOffset; + } + + /** + * @param autoCommitOffset + * + * @deprecated in favor of using {@link #ackMode} + */ + @Deprecated + public void setAutoCommitOffset(boolean autoCommitOffset) { + this.autoCommitOffset = autoCommitOffset; + } + + /** + * @return Container's ack mode. + */ + public ContainerProperties.AckMode getAckMode() { + return this.ackMode; + } + + public void setAckMode(ContainerProperties.AckMode ackMode) { + this.ackMode = ackMode; + } + + /** + * @return start offset + * + * The starting offset for new groups. Allowed values: earliest and latest. + */ + public StartOffset getStartOffset() { + return this.startOffset; + } + + public void setStartOffset(StartOffset startOffset) { + this.startOffset = startOffset; + } + + /** + * @return if resetting offset is enabled + * + * Whether to reset offsets on the consumer to the value provided by startOffset. + * Must be false if a KafkaRebalanceListener is provided. + */ + public boolean isResetOffsets() { + return this.resetOffsets; + } + + public void setResetOffsets(boolean resetOffsets) { + this.resetOffsets = resetOffsets; + } + + /** + * @return is DLQ enabled. + * + * When set to true, it enables DLQ behavior for the consumer. + * By default, messages that result in errors are forwarded to a topic named error.name-of-destination.name-of-group. + * The DLQ topic name can be configurable by setting the dlqName property. + */ + public boolean isEnableDlq() { + return this.enableDlq; + } + + public void setEnableDlq(boolean enableDlq) { + this.enableDlq = enableDlq; + } + + /** + * @return is autocommit on error in polled consumers. + * + * This property accessor is only used in polled consumers. + */ + public Boolean getAutoCommitOnError() { + return this.autoCommitOnError; + } + + /** + * + * @param autoCommitOnError commit on error in polled consumers. + * + */ + public void setAutoCommitOnError(Boolean autoCommitOnError) { + this.autoCommitOnError = autoCommitOnError; + } + + /** + * No longer used. + * @return the interval. + * @deprecated No longer used by the binder + */ + @Deprecated + public int getRecoveryInterval() { + return this.recoveryInterval; + } + + /** + * No longer used. + * @param recoveryInterval the interval. + * @deprecated No longer needed by the binder + */ + @Deprecated + public void setRecoveryInterval(int recoveryInterval) { + this.recoveryInterval = recoveryInterval; + } + + /** + * @return is auto rebalance enabled + * + * When true, topic partitions is automatically rebalanced between the members of a consumer group. + * When false, each consumer is assigned a fixed set of partitions based on spring.cloud.stream.instanceCount and spring.cloud.stream.instanceIndex. + */ + public boolean isAutoRebalanceEnabled() { + return this.autoRebalanceEnabled; + } + + public void setAutoRebalanceEnabled(boolean autoRebalanceEnabled) { + this.autoRebalanceEnabled = autoRebalanceEnabled; + } + + /** + * @return a map of configuration + * + * Map with a key/value pair containing generic Kafka consumer properties. + * In addition to having Kafka consumer properties, other configuration properties can be passed here. + */ + public Map getConfiguration() { + return this.configuration; + } + + public void setConfiguration(Map configuration) { + this.configuration = configuration; + } + + /** + * @return dlq name + * + * The name of the DLQ topic to receive the error messages. + */ + public String getDlqName() { + return this.dlqName; + } + + public void setDlqName(String dlqName) { + this.dlqName = dlqName; + } + + /** + * @return number of partitions on the DLQ topic + * + * Number of partitions to use on the DLQ. + */ + public Integer getDlqPartitions() { + return this.dlqPartitions; + } + + public void setDlqPartitions(Integer dlqPartitions) { + this.dlqPartitions = dlqPartitions; + } + + /** + * @return trusted packages + * + * List of trusted packages to provide the header mapper. + */ + public String[] getTrustedPackages() { + return this.trustedPackages; + } + + public void setTrustedPackages(String[] trustedPackages) { + this.trustedPackages = trustedPackages; + } + + /** + * @return dlq producer properties + * + * Using this, DLQ-specific producer properties can be set. + * All the properties available through kafka producer properties can be set through this property. + */ + public KafkaProducerProperties getDlqProducerProperties() { + return this.dlqProducerProperties; + } + + public void setDlqProducerProperties(KafkaProducerProperties dlqProducerProperties) { + this.dlqProducerProperties = dlqProducerProperties; + } + + /** + * @return standard headers + * + * Indicates which standard headers are populated by the inbound channel adapter. + * Allowed values: none, id, timestamp, or both. + */ + public StandardHeaders getStandardHeaders() { + return this.standardHeaders; + } + + public void setStandardHeaders(StandardHeaders standardHeaders) { + this.standardHeaders = standardHeaders; + } + + /** + * @return converter bean name + * + * The name of a bean that implements RecordMessageConverter. + */ + public String getConverterBeanName() { + return this.converterBeanName; + } + + public void setConverterBeanName(String converterBeanName) { + this.converterBeanName = converterBeanName; + } + + /** + * @return idle event interval + * + * The interval, in milliseconds, between events indicating that no messages have recently been received. + */ + public long getIdleEventInterval() { + return this.idleEventInterval; + } + + public void setIdleEventInterval(long idleEventInterval) { + this.idleEventInterval = idleEventInterval; + } + + /** + * @return is destination given through a pattern + * + * When true, the destination is treated as a regular expression Pattern used to match topic names by the broker. + */ + public boolean isDestinationIsPattern() { + return this.destinationIsPattern; + } + + public void setDestinationIsPattern(boolean destinationIsPattern) { + this.destinationIsPattern = destinationIsPattern; + } + + /** + * @return topic properties + * + * Various topic level properties. @see {@link KafkaTopicProperties} for more details. + */ + public KafkaTopicProperties getTopic() { + return this.topic; + } + + public void setTopic(KafkaTopicProperties topic) { + this.topic = topic; + } + + /** + * @return timeout in pollable consumers + * + * Timeout used for polling in pollable consumers. + */ + public long getPollTimeout() { + return this.pollTimeout; + } + + public void setPollTimeout(long pollTimeout) { + this.pollTimeout = pollTimeout; + } + + /** + * @return the transaction manager bean name. + * + * Transaction manager bean name (must be {@code KafkaAwareTransactionManager}. + */ + public String getTransactionManager() { + return this.transactionManager; + } + + public void setTransactionManager(String transactionManager) { + this.transactionManager = transactionManager; + } + + public boolean isTxCommitRecovered() { + return this.txCommitRecovered; + } + + public void setTxCommitRecovered(boolean txCommitRecovered) { + this.txCommitRecovered = txCommitRecovered; + } + + public String getCommonErrorHandlerBeanName() { + return commonErrorHandlerBeanName; + } + + public void setCommonErrorHandlerBeanName(String commonErrorHandlerBeanName) { + this.commonErrorHandlerBeanName = commonErrorHandlerBeanName; + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaExtendedBindingProperties.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaExtendedBindingProperties.java new file mode 100644 index 000000000..99357cc2a --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaExtendedBindingProperties.java @@ -0,0 +1,55 @@ +/* + * Copyright 2016-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.properties; + +import java.util.Map; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.cloud.stream.binder.AbstractExtendedBindingProperties; +import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider; + +/** + * Kafka specific extended binding properties class that extends from + * {@link AbstractExtendedBindingProperties}. + * + * @author Marius Bogoevici + * @author Gary Russell + * @author Soby Chacko + * @author Oleg Zhurakousky + */ +@ConfigurationProperties("spring.cloud.stream.kafka") +public class KafkaExtendedBindingProperties extends + AbstractExtendedBindingProperties { + + private static final String DEFAULTS_PREFIX = "spring.cloud.stream.kafka.default"; + + @Override + public String getDefaultsPrefix() { + return DEFAULTS_PREFIX; + } + + @Override + public Map getBindings() { + return this.doGetBindings(); + } + + @Override + public Class getExtendedPropertiesEntryClass() { + return KafkaBindingProperties.class; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaProducerProperties.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaProducerProperties.java new file mode 100644 index 000000000..3653c5f61 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaProducerProperties.java @@ -0,0 +1,327 @@ +/* + * Copyright 2016-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.properties; + +import java.util.HashMap; +import java.util.Map; + +import javax.validation.constraints.NotNull; + +import org.springframework.expression.Expression; + +/** + * Extended producer properties for Kafka binder. + * + * @author Marius Bogoevici + * @author Henryk Konsek + * @author Gary Russell + * @author Aldo Sinanaj + */ +public class KafkaProducerProperties { + + /** + * Upper limit, in bytes, of how much data the Kafka producer attempts to batch before sending. + */ + private int bufferSize = 16384; + + /** + * Set the compression.type producer property. Supported values are none, gzip, snappy and lz4. + * See {@link CompressionType} for more details. + */ + private CompressionType compressionType = CompressionType.none; + + /** + * Whether the producer is synchronous. + */ + private boolean sync; + + /** + * A SpEL expression evaluated against the outgoing message used to evaluate the time to wait + * for ack when synchronous publish is enabled. + */ + private Expression sendTimeoutExpression; + + /** + * How long the producer waits to allow more messages to accumulate in the same batch before sending the messages. + */ + private int batchTimeout; + + /** + * A SpEL expression evaluated against the outgoing message used to populate the key of the produced Kafka message. + */ + private Expression messageKeyExpression; + + /** + * A comma-delimited list of simple patterns to match Spring messaging headers + * to be mapped to the Kafka Headers in the ProducerRecord. + */ + private String[] headerPatterns; + + /** + * Map with a key/value pair containing generic Kafka producer properties. + */ + private Map configuration = new HashMap<>(); + + /** + * Various topic level properties. @see {@link KafkaTopicProperties} for more details. + */ + private KafkaTopicProperties topic = new KafkaTopicProperties(); + + /** + * Set to true to override the default binding destination (topic name) with the value of the + * KafkaHeaders.TOPIC message header in the outbound message. If the header is not present, + * the default binding destination is used. + */ + private boolean useTopicHeader; + + /** + * The bean name of a MessageChannel to which successful send results should be sent; + * the bean must exist in the application context. + */ + private String recordMetadataChannel; + + /** + * Transaction manager bean name - overrides the binder's transaction configuration. + */ + private String transactionManager; + + /* + * Timeout value in seconds for the duration to wait when closing the producer. + * If not set this defaults to 30 seconds. + */ + private int closeTimeout; + + /** + * Set to true to disable transactions. + */ + private boolean allowNonTransactional; + + /** + * @return buffer size + * + * Upper limit, in bytes, of how much data the Kafka producer attempts to batch before sending. + */ + public int getBufferSize() { + return this.bufferSize; + } + + public void setBufferSize(int bufferSize) { + this.bufferSize = bufferSize; + } + + /** + * @return compression type {@link CompressionType} + * + * Set the compression.type producer property. Supported values are none, gzip, snappy, lz4 and zstd. + * See {@link CompressionType} for more details. + */ + @NotNull + public CompressionType getCompressionType() { + return this.compressionType; + } + + public void setCompressionType(CompressionType compressionType) { + this.compressionType = compressionType; + } + + /** + * @return if synchronous sending is enabled + * + * Whether the producer is synchronous. + */ + public boolean isSync() { + return this.sync; + } + + public void setSync(boolean sync) { + this.sync = sync; + } + + /** + * @return timeout expression for send + * + * A SpEL expression evaluated against the outgoing message used to evaluate the time to wait + * for ack when synchronous publish is enabled. + */ + public Expression getSendTimeoutExpression() { + return this.sendTimeoutExpression; + } + + public void setSendTimeoutExpression(Expression sendTimeoutExpression) { + this.sendTimeoutExpression = sendTimeoutExpression; + } + + /** + * @return batch timeout + * + * How long the producer waits to allow more messages to accumulate in the same batch before sending the messages. + */ + public int getBatchTimeout() { + return this.batchTimeout; + } + + public void setBatchTimeout(int batchTimeout) { + this.batchTimeout = batchTimeout; + } + + /** + * @return message key expression + * + * A SpEL expression evaluated against the outgoing message used to populate the key of the produced Kafka message. + */ + public Expression getMessageKeyExpression() { + return this.messageKeyExpression; + } + + public void setMessageKeyExpression(Expression messageKeyExpression) { + this.messageKeyExpression = messageKeyExpression; + } + + /** + * @return header patterns + * + * A comma-delimited list of simple patterns to match Spring messaging headers + * to be mapped to the Kafka Headers in the ProducerRecord. + */ + public String[] getHeaderPatterns() { + return this.headerPatterns; + } + + public void setHeaderPatterns(String[] headerPatterns) { + this.headerPatterns = headerPatterns; + } + + /** + * @return map of configuration + * + * Map with a key/value pair containing generic Kafka producer properties. + */ + public Map getConfiguration() { + return this.configuration; + } + + public void setConfiguration(Map configuration) { + this.configuration = configuration; + } + + /** + * @return topic properties + * + * Various topic level properties. @see {@link KafkaTopicProperties} for more details. + */ + public KafkaTopicProperties getTopic() { + return this.topic; + } + + public void setTopic(KafkaTopicProperties topic) { + this.topic = topic; + } + + /** + * @return if using topic header + * + * Set to true to override the default binding destination (topic name) with the value of the + * KafkaHeaders.TOPIC message header in the outbound message. If the header is not present, + * the default binding destination is used. + */ + public boolean isUseTopicHeader() { + return this.useTopicHeader; + } + + public void setUseTopicHeader(boolean useTopicHeader) { + this.useTopicHeader = useTopicHeader; + } + + /** + * @return record metadata channel + * + * The bean name of a MessageChannel to which successful send results should be sent; + * the bean must exist in the application context. + */ + public String getRecordMetadataChannel() { + return this.recordMetadataChannel; + } + + public void setRecordMetadataChannel(String recordMetadataChannel) { + this.recordMetadataChannel = recordMetadataChannel; + } + + /** + * @return the transaction manager bean name. + * + * Transaction manager bean name (must be {@code KafkaAwareTransactionManager}. + */ + public String getTransactionManager() { + return this.transactionManager; + } + + public void setTransactionManager(String transactionManager) { + this.transactionManager = transactionManager; + } + + /* + * @return timeout in seconds for closing the producer + */ + public int getCloseTimeout() { + return this.closeTimeout; + } + + public void setCloseTimeout(int closeTimeout) { + this.closeTimeout = closeTimeout; + } + + public boolean isAllowNonTransactional() { + return this.allowNonTransactional; + } + + public void setAllowNonTransactional(boolean allowNonTransactional) { + this.allowNonTransactional = allowNonTransactional; + } + + /** + * Enumeration for compression types. + */ + public enum CompressionType { + + /** + * No compression. + */ + none, + + /** + * gzip based compression. + */ + gzip, + + /** + * snappy based compression. + */ + snappy, + + /** + * lz4 compression. + */ + lz4, + + /** + * zstd compression. + */ + zstd, + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaTopicProperties.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaTopicProperties.java new file mode 100644 index 000000000..e9a1425b7 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaTopicProperties.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.properties; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Properties for configuring topics. + * + * @author Aldo Sinanaj + * @since 2.2 + * + */ +public class KafkaTopicProperties { + + private Short replicationFactor; + + private Map> replicasAssignments = new HashMap<>(); + + private Map properties = new HashMap<>(); + + public Short getReplicationFactor() { + return replicationFactor; + } + + public void setReplicationFactor(Short replicationFactor) { + this.replicationFactor = replicationFactor; + } + + public Map> getReplicasAssignments() { + return replicasAssignments; + } + + public void setReplicasAssignments(Map> replicasAssignments) { + this.replicasAssignments = replicasAssignments; + } + + public Map getProperties() { + return properties; + } + + public void setProperties(Map properties) { + this.properties = properties; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/provisioning/AdminClientConfigCustomizer.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/provisioning/AdminClientConfigCustomizer.java new file mode 100644 index 000000000..25b2fe79e --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/provisioning/AdminClientConfigCustomizer.java @@ -0,0 +1,31 @@ +/* + * Copyright 2021-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.provisioning; + +import java.util.Map; + +/** + * Customizer for configuring AdminClient. + * + * @author Soby Chacko + * @since 3.1.2 + */ +@FunctionalInterface +public interface AdminClientConfigCustomizer { + + void configure(Map adminClientProperties); +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/provisioning/KafkaTopicProvisioner.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/provisioning/KafkaTopicProvisioner.java new file mode 100644 index 000000000..ade077417 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/provisioning/KafkaTopicProvisioner.java @@ -0,0 +1,661 @@ +/* + * Copyright 2014-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.provisioning; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.AlterConfigOp; +import org.apache.kafka.clients.admin.AlterConfigsResult; +import org.apache.kafka.clients.admin.Config; +import org.apache.kafka.clients.admin.ConfigEntry; +import org.apache.kafka.clients.admin.CreatePartitionsResult; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.DescribeConfigsResult; +import org.apache.kafka.clients.admin.DescribeTopicsResult; +import org.apache.kafka.clients.admin.ListTopicsResult; +import org.apache.kafka.clients.admin.NewPartitions; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.errors.TopicExistsException; +import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.cloud.stream.binder.BinderException; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaTopicProperties; +import org.springframework.cloud.stream.binder.kafka.utils.KafkaTopicUtils; +import org.springframework.cloud.stream.provisioning.ConsumerDestination; +import org.springframework.cloud.stream.provisioning.ProducerDestination; +import org.springframework.cloud.stream.provisioning.ProvisioningException; +import org.springframework.cloud.stream.provisioning.ProvisioningProvider; +import org.springframework.retry.RetryOperations; +import org.springframework.retry.backoff.ExponentialBackOffPolicy; +import org.springframework.retry.policy.SimpleRetryPolicy; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ObjectUtils; +import org.springframework.util.StringUtils; + +/** + * Kafka implementation for {@link ProvisioningProvider}. + * + * @author Soby Chacko + * @author Gary Russell + * @author Ilayaperumal Gopinathan + * @author Simon Flandergan + * @author Oleg Zhurakousky + * @author Aldo Sinanaj + */ +public class KafkaTopicProvisioner implements + // @checkstyle:off + ProvisioningProvider, ExtendedProducerProperties>, + // @checkstyle:on + InitializingBean { + + private static final Log logger = LogFactory.getLog(KafkaTopicProvisioner.class); + + private static final int DEFAULT_OPERATION_TIMEOUT = 30; + + private final KafkaBinderConfigurationProperties configurationProperties; + + private final int operationTimeout = DEFAULT_OPERATION_TIMEOUT; + + private final Map adminClientProperties; + + private RetryOperations metadataRetryOperations; + + /** + * Create an instance. + * @param kafkaBinderConfigurationProperties the binder configuration properties. + * @param kafkaProperties the boot Kafka properties used to build the + * @param adminClientConfigCustomizer to customize {@link AdminClient}. + * {@link AdminClient}. + */ + public KafkaTopicProvisioner( + KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties, + KafkaProperties kafkaProperties, + AdminClientConfigCustomizer adminClientConfigCustomizer) { + Assert.isTrue(kafkaProperties != null, "KafkaProperties cannot be null"); + this.configurationProperties = kafkaBinderConfigurationProperties; + this.adminClientProperties = kafkaProperties.buildAdminProperties(); + normalalizeBootPropsWithBinder(this.adminClientProperties, kafkaProperties, + kafkaBinderConfigurationProperties); + // If the application provides an AdminConfig customizer + // and overrides properties, that takes precedence. + if (adminClientConfigCustomizer != null) { + adminClientConfigCustomizer.configure(this.adminClientProperties); + } + } + + /** + * Mutator for metadata retry operations. + * @param metadataRetryOperations the retry configuration + */ + public void setMetadataRetryOperations(RetryOperations metadataRetryOperations) { + this.metadataRetryOperations = metadataRetryOperations; + } + + @Override + public void afterPropertiesSet() { + if (this.metadataRetryOperations == null) { + RetryTemplate retryTemplate = new RetryTemplate(); + + SimpleRetryPolicy simpleRetryPolicy = new SimpleRetryPolicy(); + simpleRetryPolicy.setMaxAttempts(10); + retryTemplate.setRetryPolicy(simpleRetryPolicy); + + ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy(); + backOffPolicy.setInitialInterval(100); + backOffPolicy.setMultiplier(2); + backOffPolicy.setMaxInterval(1000); + retryTemplate.setBackOffPolicy(backOffPolicy); + this.metadataRetryOperations = retryTemplate; + } + } + + @Override + public ProducerDestination provisionProducerDestination(final String name, + ExtendedProducerProperties properties) { + + if (logger.isInfoEnabled()) { + logger.info("Using kafka topic for outbound: " + name); + } + KafkaTopicUtils.validateTopicName(name); + try (AdminClient adminClient = createAdminClient()) { + createTopic(adminClient, name, properties.getPartitionCount(), false, + properties.getExtension().getTopic()); + int partitions = 0; + Map topicDescriptions = new HashMap<>(); + if (this.configurationProperties.isAutoCreateTopics()) { + this.metadataRetryOperations.execute(context -> { + try { + if (logger.isDebugEnabled()) { + logger.debug("Attempting to retrieve the description for the topic: " + name); + } + DescribeTopicsResult describeTopicsResult = adminClient + .describeTopics(Collections.singletonList(name)); + KafkaFuture> all = describeTopicsResult + .all(); + topicDescriptions.putAll(all.get(this.operationTimeout, TimeUnit.SECONDS)); + } + catch (Exception ex) { + throw new ProvisioningException("Problems encountered with partitions finding for: " + name, ex); + } + return null; + }); + } + TopicDescription topicDescription = topicDescriptions.get(name); + if (topicDescription != null) { + partitions = topicDescription.partitions().size(); + } + return new KafkaProducerDestination(name, partitions); + } + } + + @Override + public ConsumerDestination provisionConsumerDestination(final String name, + final String group, + ExtendedConsumerProperties properties) { + if (!properties.isMultiplex()) { + return doProvisionConsumerDestination(name, group, properties); + } + else { + String[] destinations = StringUtils.commaDelimitedListToStringArray(name); + for (String destination : destinations) { + doProvisionConsumerDestination(destination.trim(), group, properties); + } + return new KafkaConsumerDestination(name); + } + } + + private ConsumerDestination doProvisionConsumerDestination(final String name, + final String group, + ExtendedConsumerProperties properties) { + + if (properties.getExtension().isDestinationIsPattern()) { + Assert.isTrue(!properties.getExtension().isEnableDlq(), + "enableDLQ is not allowed when listening to topic patterns"); + if (logger.isDebugEnabled()) { + logger.debug("Listening to a topic pattern - " + name + + " - no provisioning performed"); + } + return new KafkaConsumerDestination(name); + } + KafkaTopicUtils.validateTopicName(name); + boolean anonymous = !StringUtils.hasText(group); + Assert.isTrue(!anonymous || !properties.getExtension().isEnableDlq(), + "DLQ support is not available for anonymous subscriptions"); + if (properties.getInstanceCount() == 0) { + throw new IllegalArgumentException("Instance count cannot be zero"); + } + int partitionCount = properties.getInstanceCount() * properties.getConcurrency(); + ConsumerDestination consumerDestination = new KafkaConsumerDestination(name); + try (AdminClient adminClient = createAdminClient()) { + createTopic(adminClient, name, partitionCount, + properties.getExtension().isAutoRebalanceEnabled(), + properties.getExtension().getTopic()); + if (this.configurationProperties.isAutoCreateTopics()) { + DescribeTopicsResult describeTopicsResult = adminClient + .describeTopics(Collections.singletonList(name)); + KafkaFuture> all = describeTopicsResult + .all(); + try { + Map topicDescriptions = all + .get(this.operationTimeout, TimeUnit.SECONDS); + TopicDescription topicDescription = topicDescriptions.get(name); + int partitions = topicDescription.partitions().size(); + consumerDestination = createDlqIfNeedBe(adminClient, name, group, + properties, anonymous, partitions); + if (consumerDestination == null) { + consumerDestination = new KafkaConsumerDestination(name, + partitions); + } + } + catch (Exception ex) { + throw new ProvisioningException("Provisioning exception encountered for " + name, ex); + } + } + } + return consumerDestination; + } + + AdminClient createAdminClient() { + return AdminClient.create(this.adminClientProperties); + } + + /** + * In general, binder properties supersede boot kafka properties. The one exception is + * the bootstrap servers. In that case, we should only override the boot properties if + * (there is a binder property AND it is a non-default value) OR (if there is no boot + * property); this is needed because the binder property never returns a null value. + * @param adminProps the admin properties to normalize. + * @param bootProps the boot kafka properties. + * @param binderProps the binder kafka properties. + */ + public static void normalalizeBootPropsWithBinder(Map adminProps, + KafkaProperties bootProps, KafkaBinderConfigurationProperties binderProps) { + // First deal with the outlier + String kafkaConnectionString = binderProps.getKafkaConnectionString(); + if (ObjectUtils + .isEmpty(adminProps.get(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG)) + || !kafkaConnectionString + .equals(binderProps.getDefaultKafkaConnectionString())) { + adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, + kafkaConnectionString); + } + // Now override any boot values with binder values + Map binderProperties = binderProps.getConfiguration(); + Set adminConfigNames = AdminClientConfig.configNames(); + binderProperties.forEach((key, value) -> { + if (key.equals(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG)) { + throw new IllegalStateException( + "Set binder bootstrap servers via the 'brokers' property, not 'configuration'"); + } + if (adminConfigNames.contains(key)) { + Object replaced = adminProps.put(key, value); + if (replaced != null && KafkaTopicProvisioner.logger.isDebugEnabled()) { + KafkaTopicProvisioner.logger.debug("Overrode boot property: [" + key + "], from: [" + + replaced + "] to: [" + value + "]"); + } + } + }); + } + + private ConsumerDestination createDlqIfNeedBe(AdminClient adminClient, String name, + String group, ExtendedConsumerProperties properties, + boolean anonymous, int partitions) { + + if (properties.getExtension().isEnableDlq() && !anonymous) { + String dlqTopic = StringUtils.hasText(properties.getExtension().getDlqName()) + ? properties.getExtension().getDlqName() + : "error." + name + "." + group; + int dlqPartitions = properties.getExtension().getDlqPartitions() == null + ? partitions + : properties.getExtension().getDlqPartitions(); + try { + final KafkaProducerProperties dlqProducerProperties = properties.getExtension().getDlqProducerProperties(); + createTopicAndPartitions(adminClient, dlqTopic, dlqPartitions, + properties.getExtension().isAutoRebalanceEnabled(), + dlqProducerProperties.getTopic()); + } + catch (Throwable throwable) { + if (throwable instanceof Error) { + throw (Error) throwable; + } + else { + throw new ProvisioningException("Provisioning exception encountered for " + name, throwable); + } + } + return new KafkaConsumerDestination(name, partitions, dlqTopic); + } + return null; + } + + private void createTopic(AdminClient adminClient, String name, int partitionCount, + boolean tolerateLowerPartitionsOnBroker, KafkaTopicProperties properties) { + try { + createTopicIfNecessary(adminClient, name, partitionCount, + tolerateLowerPartitionsOnBroker, properties); + } + // TODO: Remove catching Throwable. See this thread: + // TODO: + // https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/pull/514#discussion_r241075940 + catch (Throwable throwable) { + if (throwable instanceof Error) { + throw (Error) throwable; + } + else { + // TODO: + // https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/pull/514#discussion_r241075940 + throw new ProvisioningException("Provisioning exception encountered for " + name, throwable); + } + } + } + + private void createTopicIfNecessary(AdminClient adminClient, final String topicName, + final int partitionCount, boolean tolerateLowerPartitionsOnBroker, + KafkaTopicProperties properties) throws Throwable { + + if (this.configurationProperties.isAutoCreateTopics()) { + createTopicAndPartitions(adminClient, topicName, partitionCount, + tolerateLowerPartitionsOnBroker, properties); + } + else { + logger.info("Auto creation of topics is disabled."); + } + } + + /** + * Creates a Kafka topic if needed, or try to increase its partition count to the + * desired number. + * @param adminClient kafka admin client + * @param topicName topic name + * @param partitionCount partition count + * @param tolerateLowerPartitionsOnBroker whether lower partitions count on broker is + * tolerated ot not + * @param topicProperties kafka topic properties + * @throws Throwable from topic creation + */ + private void createTopicAndPartitions(AdminClient adminClient, final String topicName, + final int partitionCount, boolean tolerateLowerPartitionsOnBroker, + KafkaTopicProperties topicProperties) throws Throwable { + + ListTopicsResult listTopicsResult = adminClient.listTopics(); + KafkaFuture> namesFutures = listTopicsResult.names(); + + Set names = namesFutures.get(this.operationTimeout, TimeUnit.SECONDS); + if (names.contains(topicName)) { + //check if topic.properties are different from Topic Configuration in Kafka + if (this.configurationProperties.isAutoAlterTopics()) { + alterTopicConfigsIfNecessary(adminClient, topicName, topicProperties); + } + // only consider minPartitionCount for resizing if autoAddPartitions is true + int effectivePartitionCount = this.configurationProperties + .isAutoAddPartitions() + ? Math.max( + this.configurationProperties.getMinPartitionCount(), + partitionCount) + : partitionCount; + DescribeTopicsResult describeTopicsResult = adminClient + .describeTopics(Collections.singletonList(topicName)); + KafkaFuture> topicDescriptionsFuture = describeTopicsResult + .all(); + Map topicDescriptions = topicDescriptionsFuture + .get(this.operationTimeout, TimeUnit.SECONDS); + TopicDescription topicDescription = topicDescriptions.get(topicName); + int partitionSize = topicDescription.partitions().size(); + if (partitionSize < effectivePartitionCount) { + if (this.configurationProperties.isAutoAddPartitions()) { + CreatePartitionsResult partitions = adminClient + .createPartitions(Collections.singletonMap(topicName, + NewPartitions.increaseTo(effectivePartitionCount))); + partitions.all().get(this.operationTimeout, TimeUnit.SECONDS); + } + else if (tolerateLowerPartitionsOnBroker) { + logger.warn("The number of expected partitions was: " + + partitionCount + ", but " + partitionSize + + (partitionSize > 1 ? " have " : " has ") + + "been found instead." + "There will be " + + (effectivePartitionCount - partitionSize) + + " idle consumers"); + } + else { + throw new ProvisioningException( + "The number of expected partitions was: " + partitionCount + + ", but " + partitionSize + + (partitionSize > 1 ? " have " : " has ") + + "been found instead." + + "Consider either increasing the partition count of the topic or enabling " + + "`autoAddPartitions`"); + } + } + } + else { + // always consider minPartitionCount for topic creation + final int effectivePartitionCount = Math.max( + this.configurationProperties.getMinPartitionCount(), partitionCount); + this.metadataRetryOperations.execute((context) -> { + + NewTopic newTopic; + Map> replicasAssignments = topicProperties + .getReplicasAssignments(); + if (replicasAssignments != null && replicasAssignments.size() > 0) { + newTopic = new NewTopic(topicName, + topicProperties.getReplicasAssignments()); + } + else { + newTopic = new NewTopic(topicName, effectivePartitionCount, + topicProperties.getReplicationFactor() != null + ? topicProperties.getReplicationFactor() + : this.configurationProperties + .getReplicationFactor()); + } + if (topicProperties.getProperties().size() > 0) { + newTopic.configs(topicProperties.getProperties()); + } + CreateTopicsResult createTopicsResult = adminClient + .createTopics(Collections.singletonList(newTopic)); + try { + createTopicsResult.all().get(this.operationTimeout, TimeUnit.SECONDS); + } + catch (Exception ex) { + if (ex instanceof ExecutionException) { + if (ex.getCause() instanceof TopicExistsException) { + if (logger.isWarnEnabled()) { + logger.warn("Attempt to create topic: " + topicName + + ". Topic already exists."); + } + } + else { + logger.error("Failed to create topics", ex.getCause()); + throw ex.getCause(); + } + } + else { + logger.error("Failed to create topics", ex.getCause()); + throw ex.getCause(); + } + } + return null; + }); + } + } + + private void alterTopicConfigsIfNecessary(AdminClient adminClient, + String topicName, + KafkaTopicProperties topicProperties) + throws InterruptedException, ExecutionException, java.util.concurrent.TimeoutException { + ConfigResource topicConfigResource = new ConfigResource(ConfigResource.Type.TOPIC, topicName); + DescribeConfigsResult describeConfigsResult = adminClient + .describeConfigs(Collections.singletonList(topicConfigResource)); + KafkaFuture> topicConfigurationFuture = describeConfigsResult.all(); + Map topicConfigMap = topicConfigurationFuture + .get(this.operationTimeout, TimeUnit.SECONDS); + Config config = topicConfigMap.get(topicConfigResource); + final List updatedConfigEntries = topicProperties.getProperties().entrySet().stream() + .filter(propertiesEntry -> { + // Property is new and should be added + if (config.get(propertiesEntry.getKey()) == null) { + return true; + } + else { + // Property changed and should be updated + return !config.get(propertiesEntry.getKey()).value().equals(propertiesEntry.getValue()); + } + + }) + .map(propertyEntry -> new ConfigEntry(propertyEntry.getKey(), propertyEntry.getValue())) + .map(configEntry -> new AlterConfigOp(configEntry, AlterConfigOp.OpType.SET)) + .collect(Collectors.toList()); + if (!updatedConfigEntries.isEmpty()) { + if (logger.isDebugEnabled()) { + logger.debug("Attempting to alter configs " + updatedConfigEntries + " for the topic:" + topicName); + } + Map> alterConfigForTopics = new HashMap<>(); + alterConfigForTopics.put(topicConfigResource, updatedConfigEntries); + AlterConfigsResult alterConfigsResult = adminClient.incrementalAlterConfigs(alterConfigForTopics); + alterConfigsResult.all().get(this.operationTimeout, TimeUnit.SECONDS); + } + } + + /** + * Check that the topic has the expected number of partitions and return the partition information. + * @param partitionCount the expected count. + * @param tolerateLowerPartitionsOnBroker if false, throw an exception if there are not enough partitions. + * @param callable a Callable that will provide the partition information. + * @param topicName the topic./ + * @return the partition information. + */ + public Collection getPartitionsForTopic(final int partitionCount, + final boolean tolerateLowerPartitionsOnBroker, + final Callable> callable, final String topicName) { + try { + return this.metadataRetryOperations.execute((context) -> { + Collection partitions = Collections.emptyList(); + + try { + // This call may return null or throw an exception. + partitions = callable.call(); + } + catch (Exception ex) { + // The above call can potentially throw exceptions such as timeout. If + // we can determine + // that the exception was due to an unknown topic on the broker, just + // simply rethrow that. + if (ex instanceof UnknownTopicOrPartitionException) { + throw ex; + } + logger.error("Failed to obtain partition information", ex); + } + // In some cases, the above partition query may not throw an UnknownTopic..Exception for various reasons. + // For that, we are forcing another query to ensure that the topic is present on the server. + if (CollectionUtils.isEmpty(partitions)) { + try (AdminClient adminClient = AdminClient + .create(this.adminClientProperties)) { + final DescribeTopicsResult describeTopicsResult = adminClient + .describeTopics(Collections.singletonList(topicName)); + + describeTopicsResult.all().get(); + } + catch (ExecutionException ex) { + if (ex.getCause() instanceof UnknownTopicOrPartitionException) { + throw (UnknownTopicOrPartitionException) ex.getCause(); + } + else { + logger.warn("No partitions have been retrieved for the topic " + + "(" + topicName + + "). This will affect the health check."); + } + } + } + // do a sanity check on the partition set + int partitionSize = CollectionUtils.isEmpty(partitions) ? 0 : partitions.size(); + if (partitionSize < partitionCount) { + if (tolerateLowerPartitionsOnBroker) { + logger.warn("The number of expected partitions was: " + + partitionCount + ", but " + partitionSize + + (partitionSize > 1 ? " have " : " has ") + + "been found instead." + "There will be " + + (partitionCount - partitionSize) + " idle consumers"); + } + else { + throw new IllegalStateException( + "The number of expected partitions was: " + partitionCount + + ", but " + partitionSize + + (partitionSize > 1 ? " have " : " has ") + + "been found instead"); + } + } + return partitions; + }); + } + catch (Exception ex) { + logger.error("Cannot initialize Binder", ex); + throw new BinderException("Cannot initialize binder:", ex); + } + } + + private static final class KafkaProducerDestination implements ProducerDestination { + + private final String producerDestinationName; + + private final int partitions; + + KafkaProducerDestination(String destinationName, Integer partitions) { + this.producerDestinationName = destinationName; + this.partitions = partitions; + } + + @Override + public String getName() { + return this.producerDestinationName; + } + + @Override + public String getNameForPartition(int partition) { + return this.producerDestinationName; + } + + @Override + public String toString() { + return "KafkaProducerDestination{" + "producerDestinationName='" + + producerDestinationName + '\'' + ", partitions=" + partitions + '}'; + } + + } + + private static final class KafkaConsumerDestination implements ConsumerDestination { + + private final String consumerDestinationName; + + private final int partitions; + + private final String dlqName; + + KafkaConsumerDestination(String consumerDestinationName) { + this(consumerDestinationName, 0, null); + } + + KafkaConsumerDestination(String consumerDestinationName, int partitions) { + this(consumerDestinationName, partitions, null); + } + + KafkaConsumerDestination(String consumerDestinationName, Integer partitions, + String dlqName) { + this.consumerDestinationName = consumerDestinationName; + this.partitions = partitions; + this.dlqName = dlqName; + } + + @Override + public String getName() { + return this.consumerDestinationName; + } + + @Override + public String toString() { + return "KafkaConsumerDestination{" + "consumerDestinationName='" + + consumerDestinationName + '\'' + ", partitions=" + partitions + + ", dlqName='" + dlqName + '\'' + '}'; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/utils/DlqDestinationResolver.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/utils/DlqDestinationResolver.java new file mode 100644 index 000000000..cd6d18484 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/utils/DlqDestinationResolver.java @@ -0,0 +1,35 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.utils; + +import java.util.function.BiFunction; + +import org.apache.kafka.clients.consumer.ConsumerRecord; + +/** + * A {@link BiFunction} extension for defining DLQ destination resolvers. + * + * The BiFunction takes the {@link ConsumerRecord} and the exception as inputs + * and returns a topic name as the DLQ. + * + * @author Soby Chacko + * @since 3.0.9 + */ +@FunctionalInterface +public interface DlqDestinationResolver extends BiFunction, Exception, String> { + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/utils/DlqPartitionFunction.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/utils/DlqPartitionFunction.java new file mode 100644 index 000000000..8ed0534e2 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/utils/DlqPartitionFunction.java @@ -0,0 +1,76 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.utils; + +import org.apache.commons.logging.Log; +import org.apache.kafka.clients.consumer.ConsumerRecord; + +import org.springframework.lang.Nullable; + +/** + * A TriFunction that takes a consumer group, consumer record, and throwable and returns + * which partition to publish to the dead letter topic. Returning {@code null} means Kafka + * will choose the partition. + * + * @author Gary Russell + * @since 3.0 + * + */ +@FunctionalInterface +public interface DlqPartitionFunction { + + /** + * Returns the same partition as the original recor. + */ + DlqPartitionFunction ORIGINAL_PARTITION = (group, rec, ex) -> rec.partition(); + + /** + * Returns 0. + */ + DlqPartitionFunction PARTITION_ZERO = (group, rec, ex) -> 0; + + /** + * Apply the function. + * @param group the consumer group. + * @param record the consumer record. + * @param throwable the exception. + * @return the DLQ partition, or null. + */ + @Nullable + Integer apply(String group, ConsumerRecord record, Throwable throwable); + + /** + * Determine the fallback function to use based on the dlq partition count if no + * {@link DlqPartitionFunction} bean is provided. + * @param dlqPartitions the partition count. + * @param logger the logger. + * @return the fallback. + */ + static DlqPartitionFunction determineFallbackFunction(@Nullable Integer dlqPartitions, Log logger) { + if (dlqPartitions == null) { + return ORIGINAL_PARTITION; + } + else if (dlqPartitions > 1) { + logger.error("'dlqPartitions' is > 1 but a custom DlqPartitionFunction bean is not provided"); + return ORIGINAL_PARTITION; + } + else { + return PARTITION_ZERO; + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/utils/KafkaTopicUtils.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/utils/KafkaTopicUtils.java new file mode 100644 index 000000000..6fa2bcf8a --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/utils/KafkaTopicUtils.java @@ -0,0 +1,54 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.utils; + +import java.io.UnsupportedEncodingException; + +/** + * Utility methods releated to Kafka topics. + * + * @author Soby Chacko + */ +public final class KafkaTopicUtils { + + private KafkaTopicUtils() { + + } + + /** + * Validate topic name. Allowed chars are ASCII alphanumerics, '.', '_' and '-'. + * @param topicName name of the topic + */ + public static void validateTopicName(String topicName) { + try { + byte[] utf8 = topicName.getBytes("UTF-8"); + for (byte b : utf8) { + if (!((b >= 'a') && (b <= 'z') || (b >= 'A') && (b <= 'Z') + || (b >= '0') && (b <= '9') || (b == '.') || (b == '-') + || (b == '_'))) { + throw new IllegalArgumentException( + "Topic name can only have ASCII alphanumerics, '.', '_' and '-', but was: '" + + topicName + "'"); + } + } + } + catch (UnsupportedEncodingException ex) { + throw new AssertionError(ex); // Can't happen + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/test/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationPropertiesTest.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/test/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationPropertiesTest.java new file mode 100644 index 000000000..48df4962a --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/test/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationPropertiesTest.java @@ -0,0 +1,163 @@ +/* + * Copyright 2018-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.properties; + +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Map; + +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.assertj.core.util.Files; +import org.junit.Test; + +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; + +import static org.assertj.core.api.Assertions.assertThat; + +public class KafkaBinderConfigurationPropertiesTest { + + @Test + public void mergedConsumerConfigurationFiltersGroupIdFromKafkaProperties() { + KafkaProperties kafkaProperties = new KafkaProperties(); + kafkaProperties.getConsumer().setGroupId("group1"); + KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties = + new KafkaBinderConfigurationProperties(kafkaProperties); + + Map mergedConsumerConfiguration = + kafkaBinderConfigurationProperties.mergedConsumerConfiguration(); + + assertThat(mergedConsumerConfiguration).doesNotContainKeys(ConsumerConfig.GROUP_ID_CONFIG); + } + + @Test + public void mergedConsumerConfigurationFiltersEnableAutoCommitFromKafkaProperties() { + KafkaProperties kafkaProperties = new KafkaProperties(); + kafkaProperties.getConsumer().setEnableAutoCommit(true); + KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties = + new KafkaBinderConfigurationProperties(kafkaProperties); + + Map mergedConsumerConfiguration = + kafkaBinderConfigurationProperties.mergedConsumerConfiguration(); + + assertThat(mergedConsumerConfiguration).doesNotContainKeys(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG); + } + + @Test + public void mergedConsumerConfigurationFiltersGroupIdFromKafkaBinderConfigurationPropertiesConfiguration() { + KafkaProperties kafkaProperties = new KafkaProperties(); + KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties = + new KafkaBinderConfigurationProperties(kafkaProperties); + kafkaBinderConfigurationProperties + .setConfiguration(Collections.singletonMap(ConsumerConfig.GROUP_ID_CONFIG, "group1")); + + Map mergedConsumerConfiguration = kafkaBinderConfigurationProperties.mergedConsumerConfiguration(); + + assertThat(mergedConsumerConfiguration).doesNotContainKeys(ConsumerConfig.GROUP_ID_CONFIG); + } + + @Test + public void mergedConsumerConfigurationFiltersEnableAutoCommitFromKafkaBinderConfigurationPropertiesConfiguration() { + KafkaProperties kafkaProperties = new KafkaProperties(); + KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties = + new KafkaBinderConfigurationProperties(kafkaProperties); + kafkaBinderConfigurationProperties + .setConfiguration(Collections.singletonMap(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true")); + + Map mergedConsumerConfiguration = kafkaBinderConfigurationProperties.mergedConsumerConfiguration(); + + assertThat(mergedConsumerConfiguration).doesNotContainKeys(ConsumerConfig.GROUP_ID_CONFIG); + } + + @Test + public void mergedConsumerConfigurationFiltersGroupIdFromKafkaBinderConfigurationPropertiesConsumerProperties() { + KafkaProperties kafkaProperties = new KafkaProperties(); + KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties = + new KafkaBinderConfigurationProperties(kafkaProperties); + kafkaBinderConfigurationProperties + .setConsumerProperties(Collections.singletonMap(ConsumerConfig.GROUP_ID_CONFIG, "group1")); + + Map mergedConsumerConfiguration = kafkaBinderConfigurationProperties.mergedConsumerConfiguration(); + + assertThat(mergedConsumerConfiguration).doesNotContainKeys(ConsumerConfig.GROUP_ID_CONFIG); + } + + @Test + public void mergedConsumerConfigurationFiltersEnableAutoCommitFromKafkaBinderConfigurationPropertiesConsumerProps() { + KafkaProperties kafkaProperties = new KafkaProperties(); + KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties = + new KafkaBinderConfigurationProperties(kafkaProperties); + kafkaBinderConfigurationProperties + .setConsumerProperties(Collections.singletonMap(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true")); + + Map mergedConsumerConfiguration = kafkaBinderConfigurationProperties.mergedConsumerConfiguration(); + + assertThat(mergedConsumerConfiguration).doesNotContainKeys(ConsumerConfig.GROUP_ID_CONFIG); + } + + @Test + public void testCertificateFilesAreConvertedToAbsolutePathsFromClassPathResources() { + KafkaProperties kafkaProperties = new KafkaProperties(); + KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties = + new KafkaBinderConfigurationProperties(kafkaProperties); + final Map configuration = kafkaBinderConfigurationProperties.getConfiguration(); + configuration.put("ssl.truststore.location", "classpath:testclient.truststore"); + configuration.put("ssl.keystore.location", "classpath:testclient.keystore"); + + kafkaBinderConfigurationProperties.getKafkaConnectionString(); + + assertThat(configuration.get("ssl.truststore.location")) + .isEqualTo(Paths.get(System.getProperty("java.io.tmpdir"), "testclient.truststore").toString()); + assertThat(configuration.get("ssl.keystore.location")) + .isEqualTo(Paths.get(System.getProperty("java.io.tmpdir"), "testclient.keystore").toString()); + } + + @Test + public void testCertificateFilesAreConvertedToGivenAbsolutePathsFromClassPathResources() { + KafkaProperties kafkaProperties = new KafkaProperties(); + KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties = + new KafkaBinderConfigurationProperties(kafkaProperties); + final Map configuration = kafkaBinderConfigurationProperties.getConfiguration(); + configuration.put("ssl.truststore.location", "classpath:testclient.truststore"); + configuration.put("ssl.keystore.location", "classpath:testclient.keystore"); + kafkaBinderConfigurationProperties.setCertificateStoreDirectory("target"); + + kafkaBinderConfigurationProperties.getKafkaConnectionString(); + + assertThat(configuration.get("ssl.truststore.location")).isEqualTo( + Paths.get(Files.currentFolder().toString(), "target", "testclient.truststore").toString()); + assertThat(configuration.get("ssl.keystore.location")).isEqualTo( + Paths.get(Files.currentFolder().toString(), "target", "testclient.keystore").toString()); + } + + @Test + public void testCertificateFilesAreMovedForSchemaRegistryConfiguration() { + KafkaProperties kafkaProperties = new KafkaProperties(); + KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties = + new KafkaBinderConfigurationProperties(kafkaProperties); + final Map configuration = kafkaBinderConfigurationProperties.getConfiguration(); + configuration.put("schema.registry.ssl.truststore.location", "classpath:testclient.truststore"); + configuration.put("schema.registry.ssl.keystore.location", "classpath:testclient.keystore"); + kafkaBinderConfigurationProperties.setCertificateStoreDirectory("target"); + + kafkaBinderConfigurationProperties.getKafkaConnectionString(); + + assertThat(configuration.get("schema.registry.ssl.truststore.location")).isEqualTo( + Paths.get(Files.currentFolder().toString(), "target", "testclient.truststore").toString()); + assertThat(configuration.get("schema.registry.ssl.keystore.location")).isEqualTo( + Paths.get(Files.currentFolder().toString(), "target", "testclient.keystore").toString()); + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/test/java/org/springframework/cloud/stream/binder/kafka/provisioning/KafkaTopicProvisionerTests.java b/k-binder/spring-cloud-stream-binder-kafka-core/src/test/java/org/springframework/cloud/stream/binder/kafka/provisioning/KafkaTopicProvisionerTests.java new file mode 100644 index 000000000..fd630191b --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-core/src/test/java/org/springframework/cloud/stream/binder/kafka/provisioning/KafkaTopicProvisionerTests.java @@ -0,0 +1,121 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.provisioning; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.common.config.SslConfigs; +import org.apache.kafka.common.network.SslChannelBuilder; +import org.junit.Test; + +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.core.io.ClassPathResource; +import org.springframework.kafka.test.utils.KafkaTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +/** + * @author Gary Russell + * @since 2.0 + * + */ +public class KafkaTopicProvisionerTests { + + AdminClientConfigCustomizer adminClientConfigCustomizer = adminClientProperties -> adminClientProperties.put("foo", "bar"); + + @SuppressWarnings("rawtypes") + @Test + public void bootPropertiesOverriddenExceptServers() throws Exception { + KafkaProperties bootConfig = new KafkaProperties(); + bootConfig.getProperties().put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, + "PLAINTEXT"); + bootConfig.setBootstrapServers(Collections.singletonList("localhost:1234")); + KafkaBinderConfigurationProperties binderConfig = new KafkaBinderConfigurationProperties( + bootConfig); + binderConfig.getConfiguration().put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, + "SSL"); + ClassPathResource ts = new ClassPathResource("test.truststore.ks"); + binderConfig.getConfiguration().put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, + ts.getFile().getAbsolutePath()); + binderConfig.setBrokers("localhost:9092"); + KafkaTopicProvisioner provisioner = new KafkaTopicProvisioner(binderConfig, + bootConfig, adminClientConfigCustomizer); + AdminClient adminClient = provisioner.createAdminClient(); + assertThat(KafkaTestUtils.getPropertyValue(adminClient, + "client.selector.channelBuilder")).isInstanceOf(SslChannelBuilder.class); + Map configs = KafkaTestUtils.getPropertyValue(adminClient, + "client.selector.channelBuilder.configs", Map.class); + assertThat( + ((List) configs.get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG)).get(0)) + .isEqualTo("localhost:1234"); + assertThat(configs.get("foo")).isEqualTo("bar"); + adminClient.close(); + } + + @SuppressWarnings("rawtypes") + @Test + public void bootPropertiesOverriddenIncludingServers() throws Exception { + KafkaProperties bootConfig = new KafkaProperties(); + bootConfig.getProperties().put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, + "PLAINTEXT"); + bootConfig.setBootstrapServers(Collections.singletonList("localhost:9092")); + KafkaBinderConfigurationProperties binderConfig = new KafkaBinderConfigurationProperties( + bootConfig); + binderConfig.getConfiguration().put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, + "SSL"); + ClassPathResource ts = new ClassPathResource("test.truststore.ks"); + binderConfig.getConfiguration().put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, + ts.getFile().getAbsolutePath()); + binderConfig.setBrokers("localhost:1234"); + KafkaTopicProvisioner provisioner = new KafkaTopicProvisioner(binderConfig, + bootConfig, adminClientConfigCustomizer); + AdminClient adminClient = provisioner.createAdminClient(); + assertThat(KafkaTestUtils.getPropertyValue(adminClient, + "client.selector.channelBuilder")).isInstanceOf(SslChannelBuilder.class); + Map configs = KafkaTestUtils.getPropertyValue(adminClient, + "client.selector.channelBuilder.configs", Map.class); + assertThat( + ((List) configs.get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG)).get(0)) + .isEqualTo("localhost:1234"); + adminClient.close(); + } + + @Test + public void brokersInvalid() throws Exception { + KafkaProperties bootConfig = new KafkaProperties(); + KafkaBinderConfigurationProperties binderConfig = new KafkaBinderConfigurationProperties( + bootConfig); + binderConfig.getConfiguration().put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, + "localhost:1234"); + try { + new KafkaTopicProvisioner(binderConfig, bootConfig, adminClientConfigCustomizer); + fail("Expected illegal state"); + } + catch (IllegalStateException e) { + assertThat(e.getMessage()).isEqualTo( + "Set binder bootstrap servers via the 'brokers' property, not 'configuration'"); + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/test/resources/test.truststore.ks b/k-binder/spring-cloud-stream-binder-kafka-core/src/test/resources/test.truststore.ks new file mode 100644 index 000000000..24ead4bc6 Binary files /dev/null and b/k-binder/spring-cloud-stream-binder-kafka-core/src/test/resources/test.truststore.ks differ diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/test/resources/testclient.keystore b/k-binder/spring-cloud-stream-binder-kafka-core/src/test/resources/testclient.keystore new file mode 100644 index 000000000..e69de29bb diff --git a/k-binder/spring-cloud-stream-binder-kafka-core/src/test/resources/testclient.truststore b/k-binder/spring-cloud-stream-binder-kafka-core/src/test/resources/testclient.truststore new file mode 100644 index 000000000..e69de29bb diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/.jdk8 b/k-binder/spring-cloud-stream-binder-kafka-streams/.jdk8 new file mode 100644 index 000000000..e69de29bb diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/pom.xml b/k-binder/spring-cloud-stream-binder-kafka-streams/pom.xml new file mode 100644 index 000000000..44069a6e1 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/pom.xml @@ -0,0 +1,107 @@ + + + 4.0.0 + + spring-cloud-stream-binder-kafka-streams + jar + spring-cloud-stream-binder-kafka-streams + Kafka Streams Binder Implementation + + + org.springframework.cloud + spring-cloud-stream-binder-kafka-parent + 3.2.3-SNAPSHOT + + + + 1.8.2 + + + + + org.springframework.cloud + spring-cloud-stream-binder-kafka-core + + + org.springframework.boot + spring-boot-starter-actuator + true + + + org.springframework.boot + spring-boot-configuration-processor + true + + + org.springframework.boot + spring-boot-autoconfigure + true + + + org.apache.kafka + kafka-streams + + + org.springframework.kafka + spring-kafka + + + org.springframework.boot + spring-boot-test + test + + + org.springframework.kafka + spring-kafka-test + + + org.springframework.cloud + spring-cloud-stream-binder-test + test + + + org.springframework.boot + spring-boot-autoconfigure-processor + true + + + org.apache.kafka + kafka_2.13 + + + org.apache.kafka + kafka_2.13 + test + + + + org.apache.avro + avro + ${avro.version} + provided + + + + + + + org.apache.avro + avro-maven-plugin + ${avro.version} + + + generate-test-sources + + schema + + + ${project.basedir}/target/generated-test-sources + ${project.basedir}/target/generated-test-sources + ${project.basedir}/src/test/resources/avro + + + + + + + diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/AbstractKafkaStreamsBinderProcessor.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/AbstractKafkaStreamsBinderProcessor.java new file mode 100644 index 000000000..9c787b58b --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/AbstractKafkaStreamsBinderProcessor.java @@ -0,0 +1,634 @@ +/* + * Copyright 2019-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.Topology; +import org.apache.kafka.streams.errors.LogAndContinueExceptionHandler; +import org.apache.kafka.streams.errors.LogAndFailExceptionHandler; +import org.apache.kafka.streams.kstream.Consumed; +import org.apache.kafka.streams.kstream.GlobalKTable; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.processor.TimestampExtractor; +import org.apache.kafka.streams.processor.api.Processor; +import org.apache.kafka.streams.processor.api.Record; +import org.apache.kafka.streams.processor.api.RecordMetadata; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.StoreBuilder; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.boot.context.properties.bind.BindContext; +import org.springframework.boot.context.properties.bind.BindHandler; +import org.springframework.boot.context.properties.bind.Bindable; +import org.springframework.boot.context.properties.bind.PropertySourcesPlaceholdersResolver; +import org.springframework.boot.context.properties.source.ConfigurationPropertyName; +import org.springframework.boot.context.properties.source.ConfigurationPropertySources; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; +import org.springframework.cloud.stream.config.BindingProperties; +import org.springframework.cloud.stream.config.BindingServiceProperties; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.core.ResolvableType; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.MutablePropertySources; +import org.springframework.integration.support.utils.IntegrationUtils; +import org.springframework.kafka.config.KafkaStreamsConfiguration; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.config.StreamsBuilderFactoryBeanConfigurer; +import org.springframework.kafka.core.CleanupConfig; +import org.springframework.kafka.streams.RecoveringDeserializationExceptionHandler; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ObjectUtils; +import org.springframework.util.StringUtils; + +/** + * @author Soby Chacko + * @since 3.0.0 + */ +public abstract class AbstractKafkaStreamsBinderProcessor implements ApplicationContextAware { + + private static final Log LOG = LogFactory.getLog(AbstractKafkaStreamsBinderProcessor.class); + + private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; + + private final BindingServiceProperties bindingServiceProperties; + + private final KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties; + + private final CleanupConfig cleanupConfig; + + private final KeyValueSerdeResolver keyValueSerdeResolver; + + protected ConfigurableApplicationContext applicationContext; + + public AbstractKafkaStreamsBinderProcessor(BindingServiceProperties bindingServiceProperties, + KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, + KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties, + KeyValueSerdeResolver keyValueSerdeResolver, CleanupConfig cleanupConfig) { + this.bindingServiceProperties = bindingServiceProperties; + this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue; + this.kafkaStreamsExtendedBindingProperties = kafkaStreamsExtendedBindingProperties; + this.keyValueSerdeResolver = keyValueSerdeResolver; + this.cleanupConfig = cleanupConfig; + } + + @Override + public final void setApplicationContext(ApplicationContext applicationContext) + throws BeansException { + this.applicationContext = (ConfigurableApplicationContext) applicationContext; + } + + protected Topology.AutoOffsetReset getAutoOffsetReset(String inboundName, KafkaStreamsConsumerProperties extendedConsumerProperties) { + final KafkaConsumerProperties.StartOffset startOffset = extendedConsumerProperties + .getStartOffset(); + Topology.AutoOffsetReset autoOffsetReset = null; + if (startOffset != null) { + switch (startOffset) { + case earliest: + autoOffsetReset = Topology.AutoOffsetReset.EARLIEST; + break; + case latest: + autoOffsetReset = Topology.AutoOffsetReset.LATEST; + break; + default: + break; + } + } + if (extendedConsumerProperties.isResetOffsets()) { + AbstractKafkaStreamsBinderProcessor.LOG.warn("Detected resetOffsets configured on binding " + + inboundName + ". " + + "Setting resetOffsets in Kafka Streams binder does not have any effect."); + } + return autoOffsetReset; + } + + @SuppressWarnings("unchecked") + protected void handleKTableGlobalKTableInputs(Object[] arguments, int index, String input, Class parameterType, Object targetBean, + StreamsBuilderFactoryBean streamsBuilderFactoryBean, StreamsBuilder streamsBuilder, + KafkaStreamsConsumerProperties extendedConsumerProperties, + Serde keySerde, Serde valueSerde, Topology.AutoOffsetReset autoOffsetReset, boolean firstBuild) { + if (firstBuild) { + addStateStoreBeans(streamsBuilder); + } + if (parameterType.isAssignableFrom(KTable.class)) { + String materializedAs = extendedConsumerProperties.getMaterializedAs(); + String bindingDestination = this.bindingServiceProperties.getBindingDestination(input); + KTable table = getKTable(extendedConsumerProperties, streamsBuilder, keySerde, valueSerde, materializedAs, + bindingDestination, autoOffsetReset); + KTableBoundElementFactory.KTableWrapper kTableWrapper = + (KTableBoundElementFactory.KTableWrapper) targetBean; + //wrap the proxy created during the initial target type binding with real object (KTable) + kTableWrapper.wrap((KTable) table); + this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding(input, streamsBuilderFactoryBean); + this.kafkaStreamsBindingInformationCatalogue.addConsumerPropertiesPerSbfb(streamsBuilderFactoryBean, + bindingServiceProperties.getConsumerProperties(input)); + arguments[index] = table; + } + else if (parameterType.isAssignableFrom(GlobalKTable.class)) { + String materializedAs = extendedConsumerProperties.getMaterializedAs(); + String bindingDestination = this.bindingServiceProperties.getBindingDestination(input); + GlobalKTable table = getGlobalKTable(extendedConsumerProperties, streamsBuilder, keySerde, valueSerde, materializedAs, + bindingDestination, autoOffsetReset); + GlobalKTableBoundElementFactory.GlobalKTableWrapper globalKTableWrapper = + (GlobalKTableBoundElementFactory.GlobalKTableWrapper) targetBean; + //wrap the proxy created during the initial target type binding with real object (KTable) + globalKTableWrapper.wrap((GlobalKTable) table); + this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding(input, streamsBuilderFactoryBean); + this.kafkaStreamsBindingInformationCatalogue.addConsumerPropertiesPerSbfb(streamsBuilderFactoryBean, + bindingServiceProperties.getConsumerProperties(input)); + arguments[index] = table; + } + } + + @SuppressWarnings({ "unchecked" }) + protected StreamsBuilderFactoryBean buildStreamsBuilderAndRetrieveConfig(String beanNamePostPrefix, + ApplicationContext applicationContext, String inboundName, + KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties, + StreamsBuilderFactoryBeanConfigurer customizer, + ConfigurableEnvironment environment, BindingProperties bindingProperties) { + ConfigurableListableBeanFactory beanFactory = this.applicationContext + .getBeanFactory(); + + Map streamConfigGlobalProperties = applicationContext + .getBean("streamConfigGlobalProperties", Map.class); + + // Use a copy because the global configuration will be shared by multiple processors. + Map streamConfiguration = new HashMap<>(streamConfigGlobalProperties); + + if (kafkaStreamsBinderConfigurationProperties != null) { + final Map functionConfigMap = kafkaStreamsBinderConfigurationProperties.getFunctions(); + if (!CollectionUtils.isEmpty(functionConfigMap)) { + final KafkaStreamsBinderConfigurationProperties.Functions functionConfig = functionConfigMap.get(beanNamePostPrefix); + if (functionConfig != null) { + final Map functionSpecificConfig = functionConfig.getConfiguration(); + if (!CollectionUtils.isEmpty(functionSpecificConfig)) { + streamConfiguration.putAll(functionSpecificConfig); + } + + String applicationId = functionConfig.getApplicationId(); + if (!StringUtils.isEmpty(applicationId)) { + streamConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId); + } + } + } + } + + final MutablePropertySources propertySources = environment.getPropertySources(); + + if (!StringUtils.isEmpty(bindingProperties.getBinder())) { + final KafkaStreamsBinderConfigurationProperties multiBinderKafkaStreamsBinderConfigurationProperties = + applicationContext.getBean(bindingProperties.getBinder() + "-KafkaStreamsBinderConfigurationProperties", KafkaStreamsBinderConfigurationProperties.class); + String connectionString = multiBinderKafkaStreamsBinderConfigurationProperties.getKafkaConnectionString(); + if (StringUtils.isEmpty(connectionString)) { + connectionString = (String) propertySources.get(bindingProperties.getBinder() + "-kafkaStreamsBinderEnv").getProperty("spring.cloud.stream.kafka.binder.brokers"); + } + + streamConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, connectionString); + + String binderProvidedApplicationId = multiBinderKafkaStreamsBinderConfigurationProperties.getApplicationId(); + if (StringUtils.hasText(binderProvidedApplicationId)) { + streamConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, + binderProvidedApplicationId); + } + + if (multiBinderKafkaStreamsBinderConfigurationProperties + .getDeserializationExceptionHandler() == DeserializationExceptionHandler.logAndContinue) { + streamConfiguration.put( + StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, + LogAndContinueExceptionHandler.class); + } + else if (multiBinderKafkaStreamsBinderConfigurationProperties + .getDeserializationExceptionHandler() == DeserializationExceptionHandler.logAndFail) { + streamConfiguration.put( + StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, + LogAndFailExceptionHandler.class); + } + else if (multiBinderKafkaStreamsBinderConfigurationProperties + .getDeserializationExceptionHandler() == DeserializationExceptionHandler.sendToDlq) { + streamConfiguration.put( + StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, + RecoveringDeserializationExceptionHandler.class); + SendToDlqAndContinue sendToDlqAndContinue = applicationContext.getBean(SendToDlqAndContinue.class); + streamConfiguration.put(RecoveringDeserializationExceptionHandler.KSTREAM_DESERIALIZATION_RECOVERER, sendToDlqAndContinue); + } + + if (!ObjectUtils.isEmpty(multiBinderKafkaStreamsBinderConfigurationProperties.getConfiguration())) { + streamConfiguration.putAll(multiBinderKafkaStreamsBinderConfigurationProperties.getConfiguration()); + } + if (!streamConfiguration.containsKey(StreamsConfig.REPLICATION_FACTOR_CONFIG)) { + streamConfiguration.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, + (int) multiBinderKafkaStreamsBinderConfigurationProperties.getReplicationFactor()); + } + } + + //this is only used primarily for StreamListener based processors. Although in theory, functions can use it, + //it is ideal for functions to use the approach used in the above if statement by using a property like + //spring.cloud.stream.kafka.streams.binder.functions.process.configuration.num.threads (assuming that process is the function name). + KafkaStreamsConsumerProperties extendedConsumerProperties = this.kafkaStreamsExtendedBindingProperties + .getExtendedConsumerProperties(inboundName); + Map bindingConfig = extendedConsumerProperties.getConfiguration(); + Assert.state(!bindingConfig.containsKey(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG), + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG + " cannot be overridden at the binding level; " + + "use multiple binders instead"); + // We will only add the per binding configuration to the current streamConfiguration and not the global one. + streamConfiguration + .putAll(bindingConfig); + + String bindingLevelApplicationId = extendedConsumerProperties.getApplicationId(); + // override application.id if set at the individual binding level. + // We provide this for backward compatibility with StreamListener based processors. + // For function based processors see the approach used above + // (i.e. use a property like spring.cloud.stream.kafka.streams.binder.functions.process.applicationId). + if (StringUtils.hasText(bindingLevelApplicationId)) { + streamConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, + bindingLevelApplicationId); + } + + //If the application id is not set by any mechanism, then generate it. + streamConfiguration.computeIfAbsent(StreamsConfig.APPLICATION_ID_CONFIG, + k -> { + String generatedApplicationID = beanNamePostPrefix + "-applicationId"; + LOG.info("Binder Generated Kafka Streams Application ID: " + generatedApplicationID); + LOG.info("Use the binder generated application ID only for development and testing. "); + LOG.info("For production deployments, please consider explicitly setting an application ID using a configuration property."); + LOG.info("The generated applicationID is static and will be preserved over application restarts."); + return generatedApplicationID; + }); + + handleConcurrency(applicationContext, inboundName, streamConfiguration); + + // Override deserialization exception handlers per binding + final DeserializationExceptionHandler deserializationExceptionHandler = + extendedConsumerProperties.getDeserializationExceptionHandler(); + if (deserializationExceptionHandler == DeserializationExceptionHandler.logAndFail) { + streamConfiguration.put( + StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, + LogAndFailExceptionHandler.class); + } + else if (deserializationExceptionHandler == DeserializationExceptionHandler.logAndContinue) { + streamConfiguration.put( + StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, + LogAndContinueExceptionHandler.class); + } + else if (deserializationExceptionHandler == DeserializationExceptionHandler.sendToDlq) { + streamConfiguration.put( + StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, + RecoveringDeserializationExceptionHandler.class); + streamConfiguration.put(RecoveringDeserializationExceptionHandler.KSTREAM_DESERIALIZATION_RECOVERER, + applicationContext.getBean(SendToDlqAndContinue.class)); + } + else if (deserializationExceptionHandler == DeserializationExceptionHandler.skipAndContinue) { + streamConfiguration.put( + StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, + SkipAndContinueExceptionHandler.class); + } + + KafkaStreamsConfiguration kafkaStreamsConfiguration = new KafkaStreamsConfiguration(streamConfiguration); + + StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.cleanupConfig == null + ? new StreamsBuilderFactoryBean(kafkaStreamsConfiguration) + : new StreamsBuilderFactoryBean(kafkaStreamsConfiguration, + this.cleanupConfig); + + streamsBuilderFactoryBean.setAutoStartup(false); + BeanDefinition streamsBuilderBeanDefinition = BeanDefinitionBuilder + .genericBeanDefinition( + (Class) streamsBuilderFactoryBean.getClass(), + () -> streamsBuilderFactoryBean) + .getRawBeanDefinition(); + ((BeanDefinitionRegistry) beanFactory).registerBeanDefinition( + "stream-builder-" + beanNamePostPrefix, streamsBuilderBeanDefinition); + + extendedConsumerProperties.setApplicationId((String) streamConfiguration.get(StreamsConfig.APPLICATION_ID_CONFIG)); + + final StreamsBuilderFactoryBean streamsBuilderFactoryBeanFromContext = applicationContext.getBean( + "&stream-builder-" + beanNamePostPrefix, StreamsBuilderFactoryBean.class); + //At this point, the StreamsBuilderFactoryBean is created. If the users call, getObject() + //in the customizer, that should grant access to the StreamsBuilder. + if (customizer != null) { + customizer.configure(streamsBuilderFactoryBean); + } + return streamsBuilderFactoryBeanFromContext; + } + + private void handleConcurrency(ApplicationContext applicationContext, String inboundName, + Map streamConfiguration) { + // This rebinding is necessary to capture the concurrency explicitly set by the application. + // This is added to fix this issue: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/899 + org.springframework.boot.context.properties.bind.Binder explicitConcurrencyResolver = + new org.springframework.boot.context.properties.bind.Binder(ConfigurationPropertySources.get(applicationContext.getEnvironment()), + new PropertySourcesPlaceholdersResolver(applicationContext.getEnvironment()), + IntegrationUtils.getConversionService(this.applicationContext.getBeanFactory()), null); + + boolean[] concurrencyExplicitlyProvided = new boolean[] {false}; + BindHandler handler = new BindHandler() { + + @Override + public Object onSuccess(ConfigurationPropertyName name, Bindable target, + BindContext context, Object result) { + if (!concurrencyExplicitlyProvided[0]) { + + concurrencyExplicitlyProvided[0] = name.getLastElement(ConfigurationPropertyName.Form.UNIFORM) + .equals("concurrency") && + ConfigurationPropertyName.of("spring.cloud.stream.bindings." + inboundName + ".consumer").isAncestorOf(name); + } + return result; + } + }; + //Re-bind spring.cloud.stream properties to check if the application explicitly provided concurrency. + try { + explicitConcurrencyResolver.bind("spring.cloud.stream", + Bindable.ofInstance(new BindingServiceProperties()), handler); + } + catch (Exception e) { + // Ignore this exception + } + + int concurrency = this.bindingServiceProperties.getConsumerProperties(inboundName) + .getConcurrency(); + // override concurrency if set at the individual binding level. + // Concurrency will be mapped to num.stream.threads. + // This conditional also takes into account explicit concurrency settings left at the default value of 1 + // by the application to address concurrency behavior in applications with multiple processors. + // See this GH issue: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/844 + if (concurrency >= 1 && concurrencyExplicitlyProvided[0]) { + streamConfiguration.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, + concurrency); + } + } + + protected Serde getValueSerde(String inboundName, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, ResolvableType resolvableType) { + if (bindingServiceProperties.getConsumerProperties(inboundName).isUseNativeDecoding()) { + BindingProperties bindingProperties = this.bindingServiceProperties + .getBindingProperties(inboundName); + return this.keyValueSerdeResolver.getInboundValueSerde( + bindingProperties.getConsumer(), kafkaStreamsConsumerProperties, resolvableType); + } + else { + return Serdes.ByteArray(); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + protected KStream getKStream(String inboundName, BindingProperties bindingProperties, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, + StreamsBuilder streamsBuilder, Serde keySerde, Serde valueSerde, Topology.AutoOffsetReset autoOffsetReset, boolean firstBuild) { + if (firstBuild) { + addStateStoreBeans(streamsBuilder); + } + + final boolean nativeDecoding = this.bindingServiceProperties + .getConsumerProperties(inboundName).isUseNativeDecoding(); + if (nativeDecoding) { + LOG.info("Native decoding is enabled for " + inboundName + + ". Inbound deserialization done at the broker."); + } + else { + LOG.info("Native decoding is disabled for " + inboundName + + ". Inbound message conversion done by Spring Cloud Stream."); + } + + KStream stream; + final Serde valueSerdeToUse = StringUtils.hasText(kafkaStreamsConsumerProperties.getEventTypes()) ? + new Serdes.BytesSerde() : valueSerde; + final Consumed consumed = getConsumed(kafkaStreamsConsumerProperties, keySerde, valueSerdeToUse, autoOffsetReset); + + if (this.kafkaStreamsExtendedBindingProperties + .getExtendedConsumerProperties(inboundName).isDestinationIsPattern()) { + final Pattern pattern = Pattern.compile(this.bindingServiceProperties.getBindingDestination(inboundName)); + + stream = streamsBuilder.stream(pattern, consumed); + } + else { + String[] bindingTargets = StringUtils.commaDelimitedListToStringArray( + this.bindingServiceProperties.getBindingDestination(inboundName)); + stream = streamsBuilder.stream(Arrays.asList(bindingTargets), + consumed); + } + //Check to see if event type based routing is enabled. + //See this issue for more context: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1003 + if (StringUtils.hasText(kafkaStreamsConsumerProperties.getEventTypes())) { + AtomicBoolean matched = new AtomicBoolean(); + AtomicReference topicObject = new AtomicReference<>(); + AtomicReference headersObject = new AtomicReference<>(); + // Processor to retrieve the header value. + stream.process(() -> eventTypeProcessor(kafkaStreamsConsumerProperties, matched, topicObject, headersObject)); + // Branching based on event type match. + final KStream[] branch = stream.branch((key, value) -> matched.getAndSet(false)); + // Deserialize if we have a branch from above. + final KStream deserializedKStream = branch[0].mapValues(value -> valueSerde.deserializer().deserialize( + topicObject.get(), headersObject.get(), ((Bytes) value).get())); + return getkStream(bindingProperties, deserializedKStream, nativeDecoding); + } + return getkStream(bindingProperties, stream, nativeDecoding); + } + + private KStream getkStream(BindingProperties bindingProperties, KStream stream, boolean nativeDecoding) { + if (!nativeDecoding) { + stream = stream.mapValues((value) -> { + Object returnValue; + String contentType = bindingProperties.getContentType(); + if (value != null && !StringUtils.isEmpty(contentType)) { + returnValue = MessageBuilder.withPayload(value) + .setHeader(MessageHeaders.CONTENT_TYPE, contentType).build(); + } + else { + returnValue = value; + } + return returnValue; + }); + } + return stream; + } + + @SuppressWarnings("rawtypes") + private void addStateStoreBeans(StreamsBuilder streamsBuilder) { + try { + final Map storeBuilders = applicationContext.getBeansOfType(StoreBuilder.class); + if (!CollectionUtils.isEmpty(storeBuilders)) { + storeBuilders.values().forEach(storeBuilder -> { + streamsBuilder.addStateStore(storeBuilder); + if (LOG.isInfoEnabled()) { + LOG.info("state store " + storeBuilder.name() + " added to topology"); + } + }); + } + } + catch (Exception e) { + // Pass through. + } + } + + private KTable materializedAs(StreamsBuilder streamsBuilder, String destination, String storeName, + Serde k, Serde v, Topology.AutoOffsetReset autoOffsetReset, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties) { + + final Consumed consumed = getConsumed(kafkaStreamsConsumerProperties, k, v, autoOffsetReset); + return streamsBuilder.table(this.bindingServiceProperties.getBindingDestination(destination), + consumed, getMaterialized(storeName, k, v)); + } + + private Materialized> getMaterialized( + String storeName, Serde k, Serde v) { + return Materialized.>as(storeName) + .withKeySerde(k).withValueSerde(v); + } + + private GlobalKTable materializedAsGlobalKTable( + StreamsBuilder streamsBuilder, String destination, String storeName, + Serde k, Serde v, Topology.AutoOffsetReset autoOffsetReset, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties) { + final Consumed consumed = getConsumed(kafkaStreamsConsumerProperties, k, v, autoOffsetReset); + return streamsBuilder.globalTable( + this.bindingServiceProperties.getBindingDestination(destination), + consumed, + getMaterialized(storeName, k, v)); + } + + private GlobalKTable getGlobalKTable(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, + StreamsBuilder streamsBuilder, + Serde keySerde, Serde valueSerde, String materializedAs, + String bindingDestination, Topology.AutoOffsetReset autoOffsetReset) { + final Consumed consumed = getConsumed(kafkaStreamsConsumerProperties, keySerde, valueSerde, autoOffsetReset); + return materializedAs != null + ? materializedAsGlobalKTable(streamsBuilder, bindingDestination, + materializedAs, keySerde, valueSerde, autoOffsetReset, kafkaStreamsConsumerProperties) + : streamsBuilder.globalTable(bindingDestination, + consumed); + } + + private KTable getKTable(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, + StreamsBuilder streamsBuilder, Serde keySerde, + Serde valueSerde, String materializedAs, String bindingDestination, + Topology.AutoOffsetReset autoOffsetReset) { + + final Serde valueSerdeToUse = StringUtils.hasText(kafkaStreamsConsumerProperties.getEventTypes()) ? + new Serdes.BytesSerde() : valueSerde; + + final Consumed consumed = getConsumed(kafkaStreamsConsumerProperties, keySerde, valueSerdeToUse, autoOffsetReset); + + final KTable kTable = materializedAs != null + ? materializedAs(streamsBuilder, bindingDestination, materializedAs, + keySerde, valueSerdeToUse, autoOffsetReset, kafkaStreamsConsumerProperties) + : streamsBuilder.table(bindingDestination, + consumed); + if (StringUtils.hasText(kafkaStreamsConsumerProperties.getEventTypes())) { + AtomicBoolean matched = new AtomicBoolean(); + AtomicReference topicObject = new AtomicReference<>(); + AtomicReference headersObject = new AtomicReference<>(); + + final KStream stream = kTable.toStream(); + + // Processor to retrieve the header value. + stream.process(() -> eventTypeProcessor(kafkaStreamsConsumerProperties, matched, topicObject, headersObject)); + // Branching based on event type match. + final KStream[] branch = stream.branch((key, value) -> matched.getAndSet(false)); + // Deserialize if we have a branch from above. + final KStream deserializedKStream = branch[0].mapValues(value -> valueSerde.deserializer().deserialize( + topicObject.get(), headersObject.get(), ((Bytes) value).get())); + + return deserializedKStream.toTable(); + } + return kTable; + } + + private Consumed getConsumed(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, + Serde keySerde, Serde valueSerde, Topology.AutoOffsetReset autoOffsetReset) { + TimestampExtractor timestampExtractor = null; + if (!StringUtils.isEmpty(kafkaStreamsConsumerProperties.getTimestampExtractorBeanName())) { + timestampExtractor = applicationContext.getBean(kafkaStreamsConsumerProperties.getTimestampExtractorBeanName(), + TimestampExtractor.class); + } + final Consumed consumed = Consumed.with(keySerde, valueSerde) + .withOffsetResetPolicy(autoOffsetReset); + if (timestampExtractor != null) { + consumed.withTimestampExtractor(timestampExtractor); + } + if (StringUtils.hasText(kafkaStreamsConsumerProperties.getConsumedAs())) { + consumed.withName(kafkaStreamsConsumerProperties.getConsumedAs()); + } + return consumed; + } + + private Processor eventTypeProcessor(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, + AtomicBoolean matched, AtomicReference topicObject, AtomicReference headersObject) { + return new Processor() { + + org.apache.kafka.streams.processor.api.ProcessorContext context; + + @Override + public void init(org.apache.kafka.streams.processor.api.ProcessorContext context) { + Processor.super.init(context); + this.context = context; + } + + @Override + public void process(Record record) { + final Headers headers = record.headers(); + headersObject.set(headers); + final Optional optional = this.context.recordMetadata(); + if (optional.isPresent()) { + final RecordMetadata recordMetadata = optional.get(); + topicObject.set(recordMetadata.topic()); + } + final Iterable
eventTypeHeader = headers.headers(kafkaStreamsConsumerProperties.getEventTypeHeaderKey()); + if (eventTypeHeader != null && eventTypeHeader.iterator().hasNext()) { + String eventTypeFromHeader = new String(eventTypeHeader.iterator().next().value()); + final String[] eventTypesFromBinding = StringUtils.commaDelimitedListToStringArray(kafkaStreamsConsumerProperties.getEventTypes()); + for (String eventTypeFromBinding : eventTypesFromBinding) { + if (eventTypeFromHeader.equals(eventTypeFromBinding)) { + matched.set(true); + break; + } + } + } + } + + @Override + public void close() { + + } + }; + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/DeserializationExceptionHandler.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/DeserializationExceptionHandler.java new file mode 100644 index 000000000..57e1a3322 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/DeserializationExceptionHandler.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +/** + * Enumeration for various {@link org.apache.kafka.streams.errors.DeserializationExceptionHandler} types. + * + * @author Soby Chacko + * @since 3.0.0 + */ +public enum DeserializationExceptionHandler { + + /** + * Deserialization error handler with log and continue. + * See {@link org.apache.kafka.streams.errors.LogAndContinueExceptionHandler} + */ + logAndContinue, + /** + * Deserialization error handler with log and fail. + * See {@link org.apache.kafka.streams.errors.LogAndFailExceptionHandler} + */ + logAndFail, + /** + * Deserialization error handler with DLQ send. + * See {@link org.springframework.kafka.streams.RecoveringDeserializationExceptionHandler} + */ + sendToDlq, + /** + * Deserialization error handler that silently skips the error and continue. + */ + skipAndContinue; + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/EncodingDecodingBindAdviceHandler.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/EncodingDecodingBindAdviceHandler.java new file mode 100644 index 000000000..91e50adee --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/EncodingDecodingBindAdviceHandler.java @@ -0,0 +1,72 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import org.springframework.boot.context.properties.ConfigurationPropertiesBindHandlerAdvisor; +import org.springframework.boot.context.properties.bind.AbstractBindHandler; +import org.springframework.boot.context.properties.bind.BindContext; +import org.springframework.boot.context.properties.bind.BindHandler; +import org.springframework.boot.context.properties.bind.BindResult; +import org.springframework.boot.context.properties.bind.Bindable; +import org.springframework.boot.context.properties.source.ConfigurationPropertyName; + +/** + * {@link ConfigurationPropertiesBindHandlerAdvisor} to detect nativeEncoding/Decoding settings + * provided by the application explicitly. + * + * @author Soby Chacko + * @since 3.0.0 + */ +public class EncodingDecodingBindAdviceHandler implements ConfigurationPropertiesBindHandlerAdvisor { + + private boolean encodingSettingProvided; + private boolean decodingSettingProvided; + + public boolean isDecodingSettingProvided() { + return decodingSettingProvided; + } + + public boolean isEncodingSettingProvided() { + return this.encodingSettingProvided; + } + + @Override + public BindHandler apply(BindHandler bindHandler) { + BindHandler handler = new AbstractBindHandler(bindHandler) { + @Override + public Bindable onStart(ConfigurationPropertyName name, + Bindable target, BindContext context) { + final String configName = name.toString(); + if (configName.contains("use") && configName.contains("native") && + (configName.contains("encoding") || configName.contains("decoding"))) { + BindResult result = context.getBinder().bind(name, target); + if (result.isBound()) { + if (configName.contains("encoding")) { + EncodingDecodingBindAdviceHandler.this.encodingSettingProvided = true; + } + else { + EncodingDecodingBindAdviceHandler.this.decodingSettingProvided = true; + } + return target.withExistingValue(result.get()); + } + } + return bindHandler.onStart(name, target, context); + } + }; + return handler; + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/ExtendedBindingHandlerMappingsProviderAutoConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/ExtendedBindingHandlerMappingsProviderAutoConfiguration.java new file mode 100644 index 000000000..a613ad555 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/ExtendedBindingHandlerMappingsProviderAutoConfiguration.java @@ -0,0 +1,51 @@ +/* + * Copyright 2018-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.HashMap; +import java.util.Map; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.context.properties.source.ConfigurationPropertyName; +import org.springframework.cloud.stream.config.BindingHandlerAdvise.MappingsProvider; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * {@link EnableAutoConfiguration Auto-configuration} for extended binding metadata for Kafka Streams. + * + * @author Chris Bono + * @since 3.2 + */ +@Configuration(proxyBeanMethods = false) +public class ExtendedBindingHandlerMappingsProviderAutoConfiguration { + + @Bean + public MappingsProvider kafkaStreamsExtendedPropertiesDefaultMappingsProvider() { + return () -> { + Map mappings = new HashMap<>(); + mappings.put( + ConfigurationPropertyName.of("spring.cloud.stream.kafka.streams"), + ConfigurationPropertyName.of("spring.cloud.stream.kafka.streams.default")); + mappings.put( + ConfigurationPropertyName.of("spring.cloud.stream.kafka.streams.bindings"), + ConfigurationPropertyName.of("spring.cloud.stream.kafka.streams.default")); + return mappings; + }; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBinder.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBinder.java new file mode 100644 index 000000000..7269875da --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBinder.java @@ -0,0 +1,172 @@ +/* + * Copyright 2018-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.GlobalKTable; + +import org.springframework.cloud.stream.binder.AbstractBinder; +import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider; +import org.springframework.cloud.stream.binder.Binding; +import org.springframework.cloud.stream.binder.DefaultBinding; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsProducerProperties; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.util.StringUtils; + +/** + * An {@link AbstractBinder} implementation for {@link GlobalKTable}. + *

+ * Provides only consumer binding for the bound {@link GlobalKTable}. Output bindings are + * not allowed on this binder. + * + * @author Soby Chacko + * @since 2.1.0 + */ +public class GlobalKTableBinder extends + // @checkstyle:off + AbstractBinder, ExtendedConsumerProperties, ExtendedProducerProperties> + implements + ExtendedPropertiesBinder, KafkaStreamsConsumerProperties, KafkaStreamsProducerProperties> { + + // @checkstyle:on + private final KafkaStreamsBinderConfigurationProperties binderConfigurationProperties; + + private final KafkaTopicProvisioner kafkaTopicProvisioner; + + private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; + + // @checkstyle:off + private KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties = new KafkaStreamsExtendedBindingProperties(); + private final KafkaStreamsRegistry kafkaStreamsRegistry; + + // @checkstyle:on + + public GlobalKTableBinder( + KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, + KafkaTopicProvisioner kafkaTopicProvisioner, + KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, KafkaStreamsRegistry kafkaStreamsRegistry) { + this.binderConfigurationProperties = binderConfigurationProperties; + this.kafkaTopicProvisioner = kafkaTopicProvisioner; + this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue; + this.kafkaStreamsRegistry = kafkaStreamsRegistry; + } + + @Override + @SuppressWarnings("unchecked") + protected Binding> doBindConsumer(String name, + String group, GlobalKTable inputTarget, + ExtendedConsumerProperties properties) { + if (!StringUtils.hasText(group)) { + group = properties.getExtension().getApplicationId(); + } + final RetryTemplate retryTemplate = buildRetryTemplate(properties); + + final String bindingName = this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(inputTarget); + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsBindingInformationCatalogue + .getStreamsBuilderFactoryBeanPerBinding().get(bindingName); + + KafkaStreamsBinderUtils.prepareConsumerBinding(name, group, + getApplicationContext(), this.kafkaTopicProvisioner, + this.binderConfigurationProperties, properties, retryTemplate, getBeanFactory(), + this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(inputTarget), + this.kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean); + + return new DefaultBinding>(bindingName, group, inputTarget, streamsBuilderFactoryBean) { + + @Override + public boolean isInput() { + return true; + } + + @Override + public synchronized void start() { + if (!streamsBuilderFactoryBean.isRunning()) { + super.start(); + GlobalKTableBinder.this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean); + //If we cached the previous KafkaStreams object (from a binding stop on the actuator), remove it. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG); + if (kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().containsKey(applicationId)) { + kafkaStreamsBindingInformationCatalogue.removePreviousKafkaStreamsForApplicationId(applicationId); + } + } + } + + @Override + public synchronized void stop() { + if (streamsBuilderFactoryBean.isRunning()) { + final KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams(); + super.stop(); + GlobalKTableBinder.this.kafkaStreamsRegistry.unregisterKafkaStreams(kafkaStreams); + KafkaStreamsBinderUtils.closeDlqProducerFactories(kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean); + //Caching the stopped KafkaStreams for health indicator purposes on the underlying processor. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + GlobalKTableBinder.this.kafkaStreamsBindingInformationCatalogue.addPreviousKafkaStreamsForApplicationId( + (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG), kafkaStreams); + } + } + }; + } + + @Override + protected Binding> doBindProducer(String name, + GlobalKTable outboundBindTarget, + ExtendedProducerProperties properties) { + throw new UnsupportedOperationException( + "No producer level binding is allowed for GlobalKTable"); + } + + @Override + public KafkaStreamsConsumerProperties getExtendedConsumerProperties( + String channelName) { + return this.kafkaStreamsExtendedBindingProperties + .getExtendedConsumerProperties(channelName); + } + + @Override + public KafkaStreamsProducerProperties getExtendedProducerProperties( + String channelName) { + throw new UnsupportedOperationException( + "No producer binding is allowed and therefore no properties"); + } + + @Override + public String getDefaultsPrefix() { + return this.kafkaStreamsExtendedBindingProperties.getDefaultsPrefix(); + } + + @Override + public Class getExtendedPropertiesEntryClass() { + return this.kafkaStreamsExtendedBindingProperties + .getExtendedPropertiesEntryClass(); + } + + public void setKafkaStreamsExtendedBindingProperties( + KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties) { + this.kafkaStreamsExtendedBindingProperties = kafkaStreamsExtendedBindingProperties; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBinderConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBinderConfiguration.java new file mode 100644 index 000000000..6fd2f7a2d --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBinderConfiguration.java @@ -0,0 +1,94 @@ +/* + * Copyright 2018-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.Map; + +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.config.BeanFactoryPostProcessor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.cloud.stream.binder.kafka.provisioning.AdminClientConfigCustomizer; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** + * Configuration for GlobalKTable binder. + * + * @author Soby Chacko + * @since 2.1.0 + */ +@Configuration +@Import({ KafkaAutoConfiguration.class, + MultiBinderPropertiesConfiguration.class, + KafkaStreamsBinderHealthIndicatorConfiguration.class, + KafkaStreamsJaasConfiguration.class}) +public class GlobalKTableBinderConfiguration { + + @Bean + public KafkaTopicProvisioner provisioningProvider( + KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, + KafkaProperties kafkaProperties, ObjectProvider adminClientConfigCustomizer) { + return new KafkaTopicProvisioner(binderConfigurationProperties, kafkaProperties, adminClientConfigCustomizer.getIfUnique()); + } + + @Bean + public GlobalKTableBinder GlobalKTableBinder( + KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, + KafkaTopicProvisioner kafkaTopicProvisioner, + KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties, + KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, + @Qualifier("streamConfigGlobalProperties") Map streamConfigGlobalProperties, + KafkaStreamsRegistry kafkaStreamsRegistry) { + + GlobalKTableBinder globalKTableBinder = new GlobalKTableBinder(binderConfigurationProperties, + kafkaTopicProvisioner, kafkaStreamsBindingInformationCatalogue, kafkaStreamsRegistry); + globalKTableBinder.setKafkaStreamsExtendedBindingProperties( + kafkaStreamsExtendedBindingProperties); + return globalKTableBinder; + } + + @Bean + @ConditionalOnBean(name = "outerContext") + public static BeanFactoryPostProcessor outerContextBeanFactoryPostProcessor() { + return beanFactory -> { + + // It is safe to call getBean("outerContext") here, because this bean is + // registered as first + // and as independent from the parent context. + ApplicationContext outerContext = (ApplicationContext) beanFactory + .getBean("outerContext"); + beanFactory.registerSingleton( + KafkaStreamsExtendedBindingProperties.class.getSimpleName(), + outerContext.getBean(KafkaStreamsExtendedBindingProperties.class)); + beanFactory.registerSingleton( + KafkaStreamsBindingInformationCatalogue.class.getSimpleName(), + outerContext.getBean(KafkaStreamsBindingInformationCatalogue.class)); + beanFactory.registerSingleton( + KafkaStreamsRegistry.class.getSimpleName(), + outerContext.getBean(KafkaStreamsRegistry.class)); + }; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBoundElementFactory.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBoundElementFactory.java new file mode 100644 index 000000000..b7d125b57 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBoundElementFactory.java @@ -0,0 +1,134 @@ +/* + * Copyright 2018-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import org.aopalliance.intercept.MethodInterceptor; +import org.aopalliance.intercept.MethodInvocation; +import org.apache.kafka.streams.kstream.GlobalKTable; + +import org.springframework.aop.framework.ProxyFactory; +import org.springframework.cloud.stream.binder.ConsumerProperties; +import org.springframework.cloud.stream.binding.AbstractBindingTargetFactory; +import org.springframework.cloud.stream.config.BindingProperties; +import org.springframework.cloud.stream.config.BindingServiceProperties; +import org.springframework.util.Assert; + +/** + * {@link org.springframework.cloud.stream.binding.BindingTargetFactory} for + * {@link GlobalKTable} + * + * Input bindings are only created as output bindings on GlobalKTable are not allowed. + * + * @author Soby Chacko + * @since 2.1.0 + */ +public class GlobalKTableBoundElementFactory + extends AbstractBindingTargetFactory { + + private final BindingServiceProperties bindingServiceProperties; + private final EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler; + private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; + + GlobalKTableBoundElementFactory(BindingServiceProperties bindingServiceProperties, + EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler, + KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue) { + super(GlobalKTable.class); + this.bindingServiceProperties = bindingServiceProperties; + this.encodingDecodingBindAdviceHandler = encodingDecodingBindAdviceHandler; + this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue; + } + + @Override + public GlobalKTable createInput(String name) { + BindingProperties bindingProperties = this.bindingServiceProperties.getBindingProperties(name); + ConsumerProperties consumerProperties = bindingProperties.getConsumer(); + if (consumerProperties == null) { + consumerProperties = this.bindingServiceProperties.getConsumerProperties(name); + consumerProperties.setUseNativeDecoding(true); + } + else { + if (!encodingDecodingBindAdviceHandler.isDecodingSettingProvided()) { + consumerProperties.setUseNativeDecoding(true); + } + } + // Always set multiplex to true in the kafka streams binder + consumerProperties.setMultiplex(true); + + // @checkstyle:off + GlobalKTableBoundElementFactory.GlobalKTableWrapperHandler wrapper = new GlobalKTableBoundElementFactory.GlobalKTableWrapperHandler(); + // @checkstyle:on + ProxyFactory proxyFactory = new ProxyFactory( + GlobalKTableBoundElementFactory.GlobalKTableWrapper.class, + GlobalKTable.class); + proxyFactory.addAdvice(wrapper); + + final GlobalKTable proxy = (GlobalKTable) proxyFactory.getProxy(); + this.kafkaStreamsBindingInformationCatalogue.addBindingNamePerTarget(proxy, name); + return proxy; + } + + @Override + public GlobalKTable createOutput(String name) { + throw new UnsupportedOperationException( + "Outbound operations are not allowed on target type GlobalKTable"); + } + + /** + * Wrapper for GlobalKTable proxy. + */ + public interface GlobalKTableWrapper { + + void wrap(GlobalKTable delegate); + + } + + private static class GlobalKTableWrapperHandler implements + GlobalKTableBoundElementFactory.GlobalKTableWrapper, MethodInterceptor { + + private GlobalKTable delegate; + + public void wrap(GlobalKTable delegate) { + Assert.notNull(delegate, "delegate cannot be null"); + if (this.delegate == null) { + this.delegate = delegate; + } + } + + @Override + public Object invoke(MethodInvocation methodInvocation) throws Throwable { + if (methodInvocation.getMethod().getDeclaringClass() + .equals(GlobalKTable.class)) { + Assert.notNull(this.delegate, + "Trying to prepareConsumerBinding " + methodInvocation.getMethod() + + " but no delegate has been set."); + return methodInvocation.getMethod().invoke(this.delegate, + methodInvocation.getArguments()); + } + else if (methodInvocation.getMethod().getDeclaringClass() + .equals(GlobalKTableBoundElementFactory.GlobalKTableWrapper.class)) { + return methodInvocation.getMethod().invoke(this, + methodInvocation.getArguments()); + } + else { + throw new IllegalStateException( + "Only GlobalKTable method invocations are permitted"); + } + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java new file mode 100644 index 000000000..33e7890c3 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/InteractiveQueryService.java @@ -0,0 +1,292 @@ +/* + * Copyright 2018-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.KeyQueryMetadata; +import org.apache.kafka.streams.StoreQueryParameters; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.errors.InvalidStateStoreException; +import org.apache.kafka.streams.state.HostInfo; +import org.apache.kafka.streams.state.QueryableStoreType; +import org.apache.kafka.streams.state.StreamsMetadata; + +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.retry.RetryPolicy; +import org.springframework.retry.backoff.FixedBackOffPolicy; +import org.springframework.retry.policy.SimpleRetryPolicy; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.util.StringUtils; + +/** + * Services pertinent to the interactive query capabilities of Kafka Streams. This class + * provides services such as querying for a particular store, which instance is hosting a + * particular store etc. This is part of the public API of the kafka streams binder and + * the users can inject this service in their applications to make use of it. + * + * @author Soby Chacko + * @author Renwei Han + * @author Serhii Siryi + * @author Nico Pommerening + * @since 2.1.0 + */ +public class InteractiveQueryService { + + private static final Log LOG = LogFactory.getLog(InteractiveQueryService.class); + + private final KafkaStreamsRegistry kafkaStreamsRegistry; + + private final KafkaStreamsBinderConfigurationProperties binderConfigurationProperties; + + /** + * Constructor for InteractiveQueryService. + * @param kafkaStreamsRegistry holding {@link KafkaStreamsRegistry} + * @param binderConfigurationProperties kafka Streams binder configuration properties + */ + public InteractiveQueryService(KafkaStreamsRegistry kafkaStreamsRegistry, + KafkaStreamsBinderConfigurationProperties binderConfigurationProperties) { + this.kafkaStreamsRegistry = kafkaStreamsRegistry; + this.binderConfigurationProperties = binderConfigurationProperties; + } + + /** + * Retrieve and return a queryable store by name created in the application. + * @param storeName name of the queryable store + * @param storeType type of the queryable store + * @param generic queryable store + * @return queryable store. + */ + public T getQueryableStore(String storeName, QueryableStoreType storeType) { + + final RetryTemplate retryTemplate = getRetryTemplate(); + + KafkaStreams contextSpecificKafkaStreams = getThreadContextSpecificKafkaStreams(); + + return retryTemplate.execute(context -> { + T store = null; + Throwable throwable = null; + if (contextSpecificKafkaStreams != null) { + try { + store = contextSpecificKafkaStreams.store( + StoreQueryParameters.fromNameAndType( + storeName, storeType)); + } + catch (InvalidStateStoreException e) { + // pass through.. + throwable = e; + } + } + if (store != null) { + return store; + } + else if (contextSpecificKafkaStreams != null) { + LOG.warn("Store " + storeName + + " could not be found in Streams context, falling back to all known Streams instances"); + } + final Set kafkaStreams = kafkaStreamsRegistry.getKafkaStreams(); + final Iterator iterator = kafkaStreams.iterator(); + while (iterator.hasNext()) { + try { + store = iterator.next() + .store(StoreQueryParameters.fromNameAndType( + storeName, storeType)); + } + catch (InvalidStateStoreException e) { + // pass through.. + throwable = e; + } + } + if (store != null) { + return store; + } + throw new IllegalStateException( + "Error when retrieving state store: " + storeName, + throwable); + }); + } + + /** + * Retrieves the current {@link KafkaStreams} context if executing Thread is created by a Streams App (contains a matching application id in Thread's name). + * + * @return KafkaStreams instance associated with Thread + */ + private KafkaStreams getThreadContextSpecificKafkaStreams() { + return this.kafkaStreamsRegistry.getKafkaStreams().stream() + .filter(this::filterByThreadName).findAny().orElse(null); + } + + /** + * Checks if the supplied {@link KafkaStreams} instance belongs to the calling Thread by matching the Thread's name with the Streams Application Id. + * + * @param streams {@link KafkaStreams} instance to filter + * @return true if Streams Instance is associated with Thread + */ + private boolean filterByThreadName(KafkaStreams streams) { + String applicationId = kafkaStreamsRegistry.streamBuilderFactoryBean( + streams).getStreamsConfiguration() + .getProperty(StreamsConfig.APPLICATION_ID_CONFIG); + // TODO: is there some better way to find out if a Stream App created the Thread? + return Thread.currentThread().getName().contains(applicationId); + } + + /** + * Gets the current {@link HostInfo} that the calling kafka streams application is + * running on. + * + * Note that the end user applications must provide `applicaiton.server` as a + * configuration property when calling this method. If this is not available, then + * null is returned. + * @return the current {@link HostInfo} + */ + public HostInfo getCurrentHostInfo() { + Map configuration = this.binderConfigurationProperties + .getConfiguration(); + if (configuration.containsKey("application.server")) { + + String applicationServer = configuration.get("application.server"); + String[] splits = StringUtils.split(applicationServer, ":"); + + return new HostInfo(splits[0], Integer.valueOf(splits[1])); + } + return null; + } + + /** + * Gets the {@link HostInfo} where the provided store and key are hosted on. This may + * not be the current host that is running the application. Kafka Streams will look + * through all the consumer instances under the same application id and retrieves the + * proper host. + * + * Note that the end user applications must provide `application.server` as a + * configuration property for all the application instances when calling this method. + * If this is not available, then null maybe returned. + * @param generic type for key + * @param store store name + * @param key key to look for + * @param serializer {@link Serializer} for the key + * @return the {@link HostInfo} where the key for the provided store is hosted currently + */ + public HostInfo getHostInfo(String store, K key, Serializer serializer) { + final RetryTemplate retryTemplate = getRetryTemplate(); + + + return retryTemplate.execute(context -> { + Throwable throwable = null; + try { + final KeyQueryMetadata keyQueryMetadata = this.kafkaStreamsRegistry.getKafkaStreams() + .stream() + .map((k) -> Optional.ofNullable(k.queryMetadataForKey(store, key, serializer))) + .filter(Optional::isPresent).map(Optional::get).findFirst().orElse(null); + if (keyQueryMetadata != null) { + return keyQueryMetadata.activeHost(); + } + } + catch (Exception e) { + throwable = e; + } + throw new IllegalStateException( + "Error when retrieving state store", throwable != null ? throwable : new Throwable("Kafka Streams is not ready.")); + }); + } + + private RetryTemplate getRetryTemplate() { + RetryTemplate retryTemplate = new RetryTemplate(); + + KafkaStreamsBinderConfigurationProperties.StateStoreRetry stateStoreRetry = this.binderConfigurationProperties.getStateStoreRetry(); + RetryPolicy retryPolicy = new SimpleRetryPolicy(stateStoreRetry.getMaxAttempts()); + FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy(); + backOffPolicy.setBackOffPeriod(stateStoreRetry.getBackoffPeriod()); + + retryTemplate.setBackOffPolicy(backOffPolicy); + retryTemplate.setRetryPolicy(retryPolicy); + + return retryTemplate; + } + + /** + * Retrieves and returns the {@link KeyQueryMetadata} associated with the given combination of + * key and state store. If none found, it will return null. + * + * @param generic type for key + * @param store store name + * @param key key to look for + * @param serializer {@link Serializer} for the key + * @return the {@link KeyQueryMetadata} if available, null otherwise. + */ + public KeyQueryMetadata getKeyQueryMetadata(String store, K key, Serializer serializer) { + return this.kafkaStreamsRegistry.getKafkaStreams() + .stream() + .map((k) -> Optional.ofNullable(k.queryMetadataForKey(store, key, serializer))) + .filter(Optional::isPresent).map(Optional::get).findFirst().orElse(null); + } + + /** + * Retrieves and returns the {@link KafkaStreams} object that is associated with the given combination of + * key and state store. If none found, it will return null. + * + * @param generic type for key + * @param store store name + * @param key key to look for + * @param serializer {@link Serializer} for the key + * @return {@link KafkaStreams} object associated with this combination of store and key + */ + public KafkaStreams getKafkaStreams(String store, K key, Serializer serializer) { + final AtomicReference kafkaStreamsAtomicReference = new AtomicReference<>(); + this.kafkaStreamsRegistry.getKafkaStreams() + .forEach(k -> { + final KeyQueryMetadata keyQueryMetadata = k.queryMetadataForKey(store, key, serializer); + if (keyQueryMetadata != null) { + kafkaStreamsAtomicReference.set(k); + } + }); + return kafkaStreamsAtomicReference.get(); + } + + /** + * Gets the list of {@link HostInfo} where the provided store is hosted on. + * It also can include current host info. + * Kafka Streams will look through all the consumer instances under the same application id + * and retrieves all hosts info. + * + * Note that the end-user applications must provide `application.server` as a configuration property + * for all the application instances when calling this method. If this is not available, then an empty list will be returned. + * + * @param store store name + * @return the list of {@link HostInfo} where provided store is hosted on + */ + public List getAllHostsInfo(String store) { + return kafkaStreamsRegistry.getKafkaStreams() + .stream() + .flatMap(k -> k.allMetadataForStore(store).stream()) + .filter(Objects::nonNull) + .map(StreamsMetadata::hostInfo) + .collect(Collectors.toList()); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBinder.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBinder.java new file mode 100644 index 000000000..0e81494f1 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBinder.java @@ -0,0 +1,300 @@ +/* + * Copyright 2017-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.Properties; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Produced; +import org.apache.kafka.streams.processor.StreamPartitioner; + +import org.springframework.aop.framework.Advised; +import org.springframework.cloud.stream.binder.AbstractBinder; +import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider; +import org.springframework.cloud.stream.binder.Binding; +import org.springframework.cloud.stream.binder.DefaultBinding; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsProducerProperties; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.util.StringUtils; + +/** + * {@link org.springframework.cloud.stream.binder.Binder} implementation for + * {@link KStream}. This implemenation extends from the {@link AbstractBinder} directly. + *

+ * Provides both producer and consumer bindings for the bound KStream. + * + * @author Marius Bogoevici + * @author Soby Chacko + */ +class KStreamBinder extends + // @checkstyle:off + AbstractBinder, ExtendedConsumerProperties, ExtendedProducerProperties> + implements + ExtendedPropertiesBinder, KafkaStreamsConsumerProperties, KafkaStreamsProducerProperties> { + + // @checkstyle:on + + private static final Log LOG = LogFactory.getLog(KStreamBinder.class); + + private final KafkaTopicProvisioner kafkaTopicProvisioner; + + // @checkstyle:off + private KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties = new KafkaStreamsExtendedBindingProperties(); + + // @checkstyle:on + + private final KafkaStreamsBinderConfigurationProperties binderConfigurationProperties; + + private final KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate; + + private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; + + private final KeyValueSerdeResolver keyValueSerdeResolver; + + private final KafkaStreamsRegistry kafkaStreamsRegistry; + + KStreamBinder(KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, + KafkaTopicProvisioner kafkaTopicProvisioner, + KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate, + KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue, + KeyValueSerdeResolver keyValueSerdeResolver, KafkaStreamsRegistry kafkaStreamsRegistry) { + this.binderConfigurationProperties = binderConfigurationProperties; + this.kafkaTopicProvisioner = kafkaTopicProvisioner; + this.kafkaStreamsMessageConversionDelegate = kafkaStreamsMessageConversionDelegate; + this.kafkaStreamsBindingInformationCatalogue = KafkaStreamsBindingInformationCatalogue; + this.keyValueSerdeResolver = keyValueSerdeResolver; + this.kafkaStreamsRegistry = kafkaStreamsRegistry; + } + + @Override + protected Binding> doBindConsumer(String name, String group, + KStream inputTarget, + ExtendedConsumerProperties properties) { + + KStream delegate = ((KStreamBoundElementFactory.KStreamWrapperHandler) + ((Advised) inputTarget).getAdvisors()[0].getAdvice()).getDelegate(); + + this.kafkaStreamsBindingInformationCatalogue.registerConsumerProperties(delegate, properties.getExtension()); + + if (!StringUtils.hasText(group)) { + group = properties.getExtension().getApplicationId(); + } + + final RetryTemplate retryTemplate = buildRetryTemplate(properties); + + final String bindingName = this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(inputTarget); + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsBindingInformationCatalogue + .getStreamsBuilderFactoryBeanPerBinding().get(bindingName); + + KafkaStreamsBinderUtils.prepareConsumerBinding(name, group, + getApplicationContext(), this.kafkaTopicProvisioner, + this.binderConfigurationProperties, properties, retryTemplate, getBeanFactory(), + this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(inputTarget), + this.kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean); + + + return new DefaultBinding>(bindingName, group, + inputTarget, streamsBuilderFactoryBean) { + + @Override + public boolean isInput() { + return true; + } + + @Override + public synchronized void start() { + if (!streamsBuilderFactoryBean.isRunning()) { + super.start(); + KStreamBinder.this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean); + //If we cached the previous KafkaStreams object (from a binding stop on the actuator), remove it. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG); + if (kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().containsKey(applicationId)) { + kafkaStreamsBindingInformationCatalogue.removePreviousKafkaStreamsForApplicationId(applicationId); + } + } + } + + @Override + public synchronized void stop() { + if (streamsBuilderFactoryBean.isRunning()) { + final KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams(); + super.stop(); + KStreamBinder.this.kafkaStreamsRegistry.unregisterKafkaStreams(kafkaStreams); + KafkaStreamsBinderUtils.closeDlqProducerFactories(kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean); + //Caching the stopped KafkaStreams for health indicator purposes on the underlying processor. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + KStreamBinder.this.kafkaStreamsBindingInformationCatalogue.addPreviousKafkaStreamsForApplicationId( + (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG), kafkaStreams); + } + } + }; + } + + @Override + @SuppressWarnings("unchecked") + protected Binding> doBindProducer(String name, + KStream outboundBindTarget, + ExtendedProducerProperties properties) { + + ExtendedProducerProperties extendedProducerProperties = + (ExtendedProducerProperties) properties; + + this.kafkaTopicProvisioner.provisionProducerDestination(name, extendedProducerProperties); + Serde keySerde = this.keyValueSerdeResolver + .getOuboundKeySerde(properties.getExtension(), kafkaStreamsBindingInformationCatalogue.getOutboundKStreamResolvable(outboundBindTarget)); + LOG.info("Key Serde used for (outbound) " + name + ": " + keySerde.getClass().getName()); + + Serde valueSerde; + if (properties.isUseNativeEncoding()) { + valueSerde = this.keyValueSerdeResolver.getOutboundValueSerde(properties, + properties.getExtension(), kafkaStreamsBindingInformationCatalogue.getOutboundKStreamResolvable(outboundBindTarget)); + } + else { + valueSerde = Serdes.ByteArray(); + } + LOG.info("Value Serde used for (outbound) " + name + ": " + valueSerde.getClass().getName()); + + to(properties.isUseNativeEncoding(), name, outboundBindTarget, + (Serde) keySerde, (Serde) valueSerde, properties.getExtension()); + + final String bindingName = this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(outboundBindTarget); + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsBindingInformationCatalogue + .getStreamsBuilderFactoryBeanPerBinding().get(bindingName); + + // We need the application id to pass to DefaultBinding so that it won't be interpreted as an anonymous group. + // In case, if we can't find application.id (which is unlikely), we just default to bindingName. + // This will only be used for lifecycle management through actuator endpoints. + final Properties streamsConfiguration = streamsBuilderFactoryBean.getStreamsConfiguration(); + final String applicationId = streamsConfiguration != null ? (String) streamsConfiguration.get("application.id") : bindingName; + + return new DefaultBinding>(bindingName, + applicationId, outboundBindTarget, streamsBuilderFactoryBean) { + + @Override + public boolean isInput() { + return false; + } + + @Override + public synchronized void start() { + if (!streamsBuilderFactoryBean.isRunning()) { + super.start(); + KStreamBinder.this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean); + //If we cached the previous KafkaStreams object (from a binding stop on the actuator), remove it. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG); + if (kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().containsKey(applicationId)) { + kafkaStreamsBindingInformationCatalogue.removePreviousKafkaStreamsForApplicationId(applicationId); + } + } + } + + @Override + public synchronized void stop() { + if (streamsBuilderFactoryBean.isRunning()) { + final KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams(); + super.stop(); + KStreamBinder.this.kafkaStreamsRegistry.unregisterKafkaStreams(kafkaStreams); + KafkaStreamsBinderUtils.closeDlqProducerFactories(kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean); + //Caching the stopped KafkaStreams for health indicator purposes on the underlying processor + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + KStreamBinder.this.kafkaStreamsBindingInformationCatalogue.addPreviousKafkaStreamsForApplicationId( + (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG), kafkaStreams); + } + } + }; + } + + @SuppressWarnings("unchecked") + private void to(boolean isNativeEncoding, String name, + KStream outboundBindTarget, Serde keySerde, + Serde valueSerde, KafkaStreamsProducerProperties properties) { + final Produced produced = Produced.with(keySerde, valueSerde); + if (StringUtils.hasText(properties.getProducedAs())) { + produced.withName(properties.getProducedAs()); + } + StreamPartitioner streamPartitioner = null; + if (!StringUtils.isEmpty(properties.getStreamPartitionerBeanName())) { + streamPartitioner = getApplicationContext().getBean(properties.getStreamPartitionerBeanName(), + StreamPartitioner.class); + } + if (streamPartitioner != null) { + produced.withStreamPartitioner(streamPartitioner); + } + if (!isNativeEncoding) { + LOG.info("Native encoding is disabled for " + name + + ". Outbound message conversion done by Spring Cloud Stream."); + outboundBindTarget.filter((k, v) -> v == null) + .to(name, produced); + this.kafkaStreamsMessageConversionDelegate + .serializeOnOutbound(outboundBindTarget) + .to(name, produced); + } + else { + LOG.info("Native encoding is enabled for " + name + + ". Outbound serialization done at the broker."); + outboundBindTarget.to(name, produced); + } + } + + @Override + public KafkaStreamsConsumerProperties getExtendedConsumerProperties( + String channelName) { + return this.kafkaStreamsExtendedBindingProperties + .getExtendedConsumerProperties(channelName); + } + + @Override + public KafkaStreamsProducerProperties getExtendedProducerProperties( + String channelName) { + return this.kafkaStreamsExtendedBindingProperties + .getExtendedProducerProperties(channelName); + } + + public void setKafkaStreamsExtendedBindingProperties( + KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties) { + this.kafkaStreamsExtendedBindingProperties = kafkaStreamsExtendedBindingProperties; + } + + @Override + public String getDefaultsPrefix() { + return this.kafkaStreamsExtendedBindingProperties.getDefaultsPrefix(); + } + + @Override + public Class getExtendedPropertiesEntryClass() { + return this.kafkaStreamsExtendedBindingProperties + .getExtendedPropertiesEntryClass(); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBinderConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBinderConfiguration.java new file mode 100644 index 000000000..2341759c7 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBinderConfiguration.java @@ -0,0 +1,98 @@ +/* + * Copyright 2017-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.config.BeanFactoryPostProcessor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.cloud.stream.binder.kafka.provisioning.AdminClientConfigCustomizer; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** + * Configuration for KStream binder. + * + * @author Marius Bogoevici + * @author Gary Russell + * @author Soby Chacko + */ +@Configuration +@Import({ KafkaAutoConfiguration.class, + MultiBinderPropertiesConfiguration.class, + KafkaStreamsBinderHealthIndicatorConfiguration.class, + KafkaStreamsJaasConfiguration.class}) +public class KStreamBinderConfiguration { + + @Bean + public KafkaTopicProvisioner provisioningProvider( + KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties, + KafkaProperties kafkaProperties, ObjectProvider adminClientConfigCustomizer) { + return new KafkaTopicProvisioner(kafkaStreamsBinderConfigurationProperties, + kafkaProperties, adminClientConfigCustomizer.getIfUnique()); + } + + @Bean + public KStreamBinder kStreamBinder( + KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, + KafkaTopicProvisioner kafkaTopicProvisioner, + KafkaStreamsMessageConversionDelegate KafkaStreamsMessageConversionDelegate, + KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue, + KeyValueSerdeResolver keyValueSerdeResolver, + KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties, KafkaStreamsRegistry kafkaStreamsRegistry) { + KStreamBinder kStreamBinder = new KStreamBinder(binderConfigurationProperties, + kafkaTopicProvisioner, KafkaStreamsMessageConversionDelegate, + KafkaStreamsBindingInformationCatalogue, keyValueSerdeResolver, kafkaStreamsRegistry); + kStreamBinder.setKafkaStreamsExtendedBindingProperties( + kafkaStreamsExtendedBindingProperties); + return kStreamBinder; + } + + @Bean + @ConditionalOnBean(name = "outerContext") + public static BeanFactoryPostProcessor outerContextBeanFactoryPostProcessor() { + return beanFactory -> { + + // It is safe to call getBean("outerContext") here, because this bean is + // registered as first + // and as independent from the parent context. + ApplicationContext outerContext = (ApplicationContext) beanFactory + .getBean("outerContext"); + beanFactory.registerSingleton( + KafkaStreamsMessageConversionDelegate.class.getSimpleName(), + outerContext.getBean(KafkaStreamsMessageConversionDelegate.class)); + beanFactory.registerSingleton( + KafkaStreamsBindingInformationCatalogue.class.getSimpleName(), + outerContext.getBean(KafkaStreamsBindingInformationCatalogue.class)); + beanFactory.registerSingleton(KeyValueSerdeResolver.class.getSimpleName(), + outerContext.getBean(KeyValueSerdeResolver.class)); + beanFactory.registerSingleton( + KafkaStreamsExtendedBindingProperties.class.getSimpleName(), + outerContext.getBean(KafkaStreamsExtendedBindingProperties.class)); + beanFactory.registerSingleton( + KafkaStreamsRegistry.class.getSimpleName(), + outerContext.getBean(KafkaStreamsRegistry.class)); + }; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBoundElementFactory.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBoundElementFactory.java new file mode 100644 index 000000000..d484a5ba7 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBoundElementFactory.java @@ -0,0 +1,156 @@ +/* + * Copyright 2017-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import org.aopalliance.intercept.MethodInterceptor; +import org.aopalliance.intercept.MethodInvocation; +import org.apache.kafka.streams.kstream.KStream; + +import org.springframework.aop.framework.ProxyFactory; +import org.springframework.cloud.stream.binder.ConsumerProperties; +import org.springframework.cloud.stream.binder.ProducerProperties; +import org.springframework.cloud.stream.binding.AbstractBindingTargetFactory; +import org.springframework.cloud.stream.config.BindingProperties; +import org.springframework.cloud.stream.config.BindingServiceProperties; +import org.springframework.util.Assert; + +/** + * {@link org.springframework.cloud.stream.binding.BindingTargetFactory} + * for{@link KStream}. + * + * The implementation creates proxies for both input and output binding. The actual target + * will be created downstream through further binding process. + * + * @author Marius Bogoevici + * @author Soby Chacko + */ +class KStreamBoundElementFactory extends AbstractBindingTargetFactory { + + private final BindingServiceProperties bindingServiceProperties; + + private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; + private final EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler; + + KStreamBoundElementFactory(BindingServiceProperties bindingServiceProperties, + KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue, + EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler) { + super(KStream.class); + this.bindingServiceProperties = bindingServiceProperties; + this.kafkaStreamsBindingInformationCatalogue = KafkaStreamsBindingInformationCatalogue; + this.encodingDecodingBindAdviceHandler = encodingDecodingBindAdviceHandler; + } + + @Override + public KStream createInput(String name) { + BindingProperties bindingProperties = this.bindingServiceProperties.getBindingProperties(name); + ConsumerProperties consumerProperties = bindingProperties.getConsumer(); + if (consumerProperties == null) { + consumerProperties = this.bindingServiceProperties.getConsumerProperties(name); + consumerProperties.setUseNativeDecoding(true); + } + else { + if (!encodingDecodingBindAdviceHandler.isDecodingSettingProvided()) { + consumerProperties.setUseNativeDecoding(true); + } + } + // Always set multiplex to true in the kafka streams binder + consumerProperties.setMultiplex(true); + return createProxyForKStream(name); + } + + @Override + @SuppressWarnings("unchecked") + public KStream createOutput(final String name) { + + BindingProperties bindingProperties = this.bindingServiceProperties.getBindingProperties(name); + ProducerProperties producerProperties = bindingProperties.getProducer(); + if (producerProperties == null) { + producerProperties = this.bindingServiceProperties.getProducerProperties(name); + producerProperties.setUseNativeEncoding(true); + } + else { + if (!encodingDecodingBindAdviceHandler.isEncodingSettingProvided()) { + producerProperties.setUseNativeEncoding(true); + } + } + return createProxyForKStream(name); + } + + private KStream createProxyForKStream(String name) { + KStreamWrapperHandler wrapper = new KStreamWrapperHandler(); + ProxyFactory proxyFactory = new ProxyFactory(KStreamWrapper.class, KStream.class); + proxyFactory.addAdvice(wrapper); + + KStream proxy = (KStream) proxyFactory.getProxy(); + + // Add the binding properties to the catalogue for later retrieval during further + // binding steps downstream. + BindingProperties bindingProperties = this.bindingServiceProperties + .getBindingProperties(name); + this.kafkaStreamsBindingInformationCatalogue.registerBindingProperties(proxy, + bindingProperties); + this.kafkaStreamsBindingInformationCatalogue.addBindingNamePerTarget(proxy, name); + return proxy; + } + + /** + * Wrapper object for KStream proxy. + */ + public interface KStreamWrapper { + + void wrap(KStream delegate); + + } + + static class KStreamWrapperHandler + implements KStreamWrapper, MethodInterceptor { + + private KStream delegate; + + public void wrap(KStream delegate) { + Assert.notNull(delegate, "delegate cannot be null"); + if (this.delegate == null) { + this.delegate = delegate; + } + } + + @Override + public Object invoke(MethodInvocation methodInvocation) throws Throwable { + if (methodInvocation.getMethod().getDeclaringClass().equals(KStream.class)) { + Assert.notNull(this.delegate, + "Trying to prepareConsumerBinding " + methodInvocation.getMethod() + + " but no delegate has been set."); + return methodInvocation.getMethod().invoke(this.delegate, + methodInvocation.getArguments()); + } + else if (methodInvocation.getMethod().getDeclaringClass() + .equals(KStreamWrapper.class)) { + return methodInvocation.getMethod().invoke(this, + methodInvocation.getArguments()); + } + else { + throw new IllegalStateException( + "Only KStream method invocations are permitted"); + } + } + + KStream getDelegate() { + return delegate; + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerParameterAdapter.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerParameterAdapter.java new file mode 100644 index 000000000..d2645c957 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerParameterAdapter.java @@ -0,0 +1,66 @@ +/* + * Copyright 2017-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import org.apache.kafka.streams.kstream.KStream; + +import org.springframework.cloud.stream.binding.StreamListenerParameterAdapter; +import org.springframework.core.MethodParameter; +import org.springframework.core.ResolvableType; + +/** + * {@link StreamListenerParameterAdapter} for KStream. + * + * @author Marius Bogoevici + * @author Soby Chacko + */ +class KStreamStreamListenerParameterAdapter + implements StreamListenerParameterAdapter, KStream> { + + private final KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate; + + private final KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue; + + KStreamStreamListenerParameterAdapter( + KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate, + KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue) { + this.kafkaStreamsMessageConversionDelegate = kafkaStreamsMessageConversionDelegate; + this.KafkaStreamsBindingInformationCatalogue = KafkaStreamsBindingInformationCatalogue; + } + + @Override + public boolean supports(Class bindingTargetType, MethodParameter methodParameter) { + return KafkaStreamsBinderUtils.supportsKStream(methodParameter, bindingTargetType); + } + + @Override + @SuppressWarnings("unchecked") + public KStream adapt(KStream bindingTarget, MethodParameter parameter) { + ResolvableType resolvableType = ResolvableType.forMethodParameter(parameter); + final Class valueClass = (resolvableType.getGeneric(1).getRawClass() != null) + ? (resolvableType.getGeneric(1).getRawClass()) : Object.class; + if (this.KafkaStreamsBindingInformationCatalogue + .isUseNativeDecoding(bindingTarget)) { + return bindingTarget; + } + else { + return this.kafkaStreamsMessageConversionDelegate + .deserializeOnInbound(valueClass, bindingTarget); + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerResultAdapter.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerResultAdapter.java new file mode 100644 index 000000000..6ffce5732 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamStreamListenerResultAdapter.java @@ -0,0 +1,58 @@ +/* + * Copyright 2017-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.io.Closeable; +import java.io.IOException; + +import org.apache.kafka.streams.kstream.KStream; + +import org.springframework.cloud.stream.binding.StreamListenerResultAdapter; + +/** + * {@link StreamListenerResultAdapter} for KStream. + * + * @author Marius Bogoevici + * @author Soby Chacko + */ +class KStreamStreamListenerResultAdapter implements + StreamListenerResultAdapter { + + @Override + public boolean supports(Class resultType, Class boundElement) { + return KStream.class.isAssignableFrom(resultType) + && KStream.class.isAssignableFrom(boundElement); + } + + @Override + @SuppressWarnings("unchecked") + public Closeable adapt(KStream streamListenerResult, + KStreamBoundElementFactory.KStreamWrapper boundElement) { + boundElement.wrap(streamListenerResult); + return new NoOpCloseable(); + } + + private static final class NoOpCloseable implements Closeable { + + @Override + public void close() throws IOException { + + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBinder.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBinder.java new file mode 100644 index 000000000..a5dcd9c09 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBinder.java @@ -0,0 +1,172 @@ +/* + * Copyright 2018-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.KTable; + +import org.springframework.cloud.stream.binder.AbstractBinder; +import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider; +import org.springframework.cloud.stream.binder.Binding; +import org.springframework.cloud.stream.binder.DefaultBinding; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsProducerProperties; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.util.StringUtils; + +/** + * {@link org.springframework.cloud.stream.binder.Binder} implementation for + * {@link KTable}. This implemenation extends from the {@link AbstractBinder} directly. + * + * Provides only consumer binding for the bound KTable as output bindings are not allowed + * on it. + * + * @author Soby Chacko + */ +class KTableBinder extends + AbstractBinder, ExtendedConsumerProperties, ExtendedProducerProperties> + implements + ExtendedPropertiesBinder, KafkaStreamsConsumerProperties, KafkaStreamsProducerProperties> { + + private final KafkaStreamsBinderConfigurationProperties binderConfigurationProperties; + + private final KafkaTopicProvisioner kafkaTopicProvisioner; + + private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; + + // @checkstyle:off + private KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties = new KafkaStreamsExtendedBindingProperties(); + + // @checkstyle:on + + private final KafkaStreamsRegistry kafkaStreamsRegistry; + + KTableBinder(KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, + KafkaTopicProvisioner kafkaTopicProvisioner, + KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue, KafkaStreamsRegistry kafkaStreamsRegistry) { + this.binderConfigurationProperties = binderConfigurationProperties; + this.kafkaTopicProvisioner = kafkaTopicProvisioner; + this.kafkaStreamsBindingInformationCatalogue = KafkaStreamsBindingInformationCatalogue; + this.kafkaStreamsRegistry = kafkaStreamsRegistry; + } + + @Override + @SuppressWarnings("unchecked") + protected Binding> doBindConsumer(String name, String group, + KTable inputTarget, + // @checkstyle:off + ExtendedConsumerProperties properties) { + // @checkstyle:on + if (!StringUtils.hasText(group)) { + group = properties.getExtension().getApplicationId(); + } + + final RetryTemplate retryTemplate = buildRetryTemplate(properties); + + final String bindingName = this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(inputTarget); + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsBindingInformationCatalogue + .getStreamsBuilderFactoryBeanPerBinding().get(bindingName); + + KafkaStreamsBinderUtils.prepareConsumerBinding(name, group, + getApplicationContext(), this.kafkaTopicProvisioner, + this.binderConfigurationProperties, properties, retryTemplate, getBeanFactory(), + this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(inputTarget), + this.kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean); + + return new DefaultBinding>(bindingName, group, inputTarget, streamsBuilderFactoryBean) { + + @Override + public boolean isInput() { + return true; + } + + @Override + public synchronized void start() { + if (!streamsBuilderFactoryBean.isRunning()) { + super.start(); + KTableBinder.this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean); + //If we cached the previous KafkaStreams object (from a binding stop on the actuator), remove it. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG); + if (kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().containsKey(applicationId)) { + kafkaStreamsBindingInformationCatalogue.removePreviousKafkaStreamsForApplicationId(applicationId); + } + } + } + + @Override + public synchronized void stop() { + if (streamsBuilderFactoryBean.isRunning()) { + final KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams(); + super.stop(); + KTableBinder.this.kafkaStreamsRegistry.unregisterKafkaStreams(kafkaStreams); + KafkaStreamsBinderUtils.closeDlqProducerFactories(kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean); + //Caching the stopped KafkaStreams for health indicator purposes on the underlying processor. + //See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + KTableBinder.this.kafkaStreamsBindingInformationCatalogue.addPreviousKafkaStreamsForApplicationId( + (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG), kafkaStreams); + } + } + }; + } + + @Override + protected Binding> doBindProducer(String name, + KTable outboundBindTarget, + ExtendedProducerProperties properties) { + throw new UnsupportedOperationException( + "No producer level binding is allowed for KTable"); + } + + @Override + public KafkaStreamsConsumerProperties getExtendedConsumerProperties( + String channelName) { + return this.kafkaStreamsExtendedBindingProperties + .getExtendedConsumerProperties(channelName); + } + + @Override + public KafkaStreamsProducerProperties getExtendedProducerProperties( + String channelName) { + return this.kafkaStreamsExtendedBindingProperties + .getExtendedProducerProperties(channelName); + } + + @Override + public String getDefaultsPrefix() { + return this.kafkaStreamsExtendedBindingProperties.getDefaultsPrefix(); + } + + @Override + public Class getExtendedPropertiesEntryClass() { + return this.kafkaStreamsExtendedBindingProperties + .getExtendedPropertiesEntryClass(); + } + + public void setKafkaStreamsExtendedBindingProperties( + KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties) { + this.kafkaStreamsExtendedBindingProperties = kafkaStreamsExtendedBindingProperties; + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBinderConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBinderConfiguration.java new file mode 100644 index 000000000..ef0d5c94a --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBinderConfiguration.java @@ -0,0 +1,93 @@ +/* + * Copyright 2018-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.Map; + +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.config.BeanFactoryPostProcessor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.cloud.stream.binder.kafka.provisioning.AdminClientConfigCustomizer; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** + * Configuration for KTable binder. + * + * @author Soby Chacko + */ +@SuppressWarnings("ALL") +@Configuration +@Import({ KafkaAutoConfiguration.class, + MultiBinderPropertiesConfiguration.class, + KafkaStreamsBinderHealthIndicatorConfiguration.class, + KafkaStreamsJaasConfiguration.class}) +public class KTableBinderConfiguration { + + @Bean + public KafkaTopicProvisioner provisioningProvider( + KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, + KafkaProperties kafkaProperties, ObjectProvider adminClientConfigCustomizer) { + return new KafkaTopicProvisioner(binderConfigurationProperties, kafkaProperties, adminClientConfigCustomizer.getIfUnique()); + } + + @Bean + public KTableBinder kTableBinder( + KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, + KafkaTopicProvisioner kafkaTopicProvisioner, + KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties, + KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, + @Qualifier("streamConfigGlobalProperties") Map streamConfigGlobalProperties, + KafkaStreamsRegistry kafkaStreamsRegistry) { + KTableBinder kTableBinder = new KTableBinder(binderConfigurationProperties, + kafkaTopicProvisioner, kafkaStreamsBindingInformationCatalogue, kafkaStreamsRegistry); + kTableBinder.setKafkaStreamsExtendedBindingProperties(kafkaStreamsExtendedBindingProperties); + return kTableBinder; + } + + @Bean + @ConditionalOnBean(name = "outerContext") + public static BeanFactoryPostProcessor outerContextBeanFactoryPostProcessor() { + return beanFactory -> { + + // It is safe to call getBean("outerContext") here, because this bean is + // registered as first + // and as independent from the parent context. + ApplicationContext outerContext = (ApplicationContext) beanFactory + .getBean("outerContext"); + beanFactory.registerSingleton( + KafkaStreamsExtendedBindingProperties.class.getSimpleName(), + outerContext.getBean(KafkaStreamsExtendedBindingProperties.class)); + beanFactory.registerSingleton( + KafkaStreamsBindingInformationCatalogue.class.getSimpleName(), + outerContext.getBean(KafkaStreamsBindingInformationCatalogue.class)); + beanFactory.registerSingleton( + KafkaStreamsRegistry.class.getSimpleName(), + outerContext.getBean(KafkaStreamsRegistry.class)); + }; + } + + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBoundElementFactory.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBoundElementFactory.java new file mode 100644 index 000000000..2dc7a90a7 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBoundElementFactory.java @@ -0,0 +1,129 @@ +/* + * Copyright 2018-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import org.aopalliance.intercept.MethodInterceptor; +import org.aopalliance.intercept.MethodInvocation; +import org.apache.kafka.streams.kstream.KTable; + +import org.springframework.aop.framework.ProxyFactory; +import org.springframework.cloud.stream.binder.ConsumerProperties; +import org.springframework.cloud.stream.binding.AbstractBindingTargetFactory; +import org.springframework.cloud.stream.config.BindingProperties; +import org.springframework.cloud.stream.config.BindingServiceProperties; +import org.springframework.util.Assert; + +/** + * {@link org.springframework.cloud.stream.binding.BindingTargetFactory} for + * {@link KTable} + * + * Input bindings are only created as output bindings on KTable are not allowed. + * + * @author Soby Chacko + */ +class KTableBoundElementFactory extends AbstractBindingTargetFactory { + + private final BindingServiceProperties bindingServiceProperties; + private final EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler; + private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; + + KTableBoundElementFactory(BindingServiceProperties bindingServiceProperties, + EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler, + KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue) { + super(KTable.class); + this.bindingServiceProperties = bindingServiceProperties; + this.encodingDecodingBindAdviceHandler = encodingDecodingBindAdviceHandler; + this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue; + } + + @Override + public KTable createInput(String name) { + BindingProperties bindingProperties = this.bindingServiceProperties.getBindingProperties(name); + ConsumerProperties consumerProperties = bindingProperties.getConsumer(); + if (consumerProperties == null) { + consumerProperties = this.bindingServiceProperties.getConsumerProperties(name); + consumerProperties.setUseNativeDecoding(true); + } + else { + if (!encodingDecodingBindAdviceHandler.isDecodingSettingProvided()) { + consumerProperties.setUseNativeDecoding(true); + } + } + // Always set multiplex to true in the kafka streams binder + consumerProperties.setMultiplex(true); + + KTableBoundElementFactory.KTableWrapperHandler wrapper = new KTableBoundElementFactory.KTableWrapperHandler(); + ProxyFactory proxyFactory = new ProxyFactory( + KTableBoundElementFactory.KTableWrapper.class, KTable.class); + proxyFactory.addAdvice(wrapper); + + final KTable proxy = (KTable) proxyFactory.getProxy(); + this.kafkaStreamsBindingInformationCatalogue.addBindingNamePerTarget(proxy, name); + return proxy; + } + + @Override + @SuppressWarnings("unchecked") + public KTable createOutput(final String name) { + throw new UnsupportedOperationException( + "Outbound operations are not allowed on target type KTable"); + } + + /** + * Wrapper for KTable proxy. + */ + public interface KTableWrapper { + + void wrap(KTable delegate); + + } + + private static class KTableWrapperHandler + implements KTableBoundElementFactory.KTableWrapper, MethodInterceptor { + + private KTable delegate; + + public void wrap(KTable delegate) { + Assert.notNull(delegate, "delegate cannot be null"); + if (this.delegate == null) { + this.delegate = delegate; + } + } + + @Override + public Object invoke(MethodInvocation methodInvocation) throws Throwable { + if (methodInvocation.getMethod().getDeclaringClass().equals(KTable.class)) { + Assert.notNull(this.delegate, + "Trying to prepareConsumerBinding " + methodInvocation.getMethod() + + " but no delegate has been set."); + return methodInvocation.getMethod().invoke(this.delegate, + methodInvocation.getArguments()); + } + else if (methodInvocation.getMethod().getDeclaringClass() + .equals(KTableBoundElementFactory.KTableWrapper.class)) { + return methodInvocation.getMethod().invoke(this, + methodInvocation.getArguments()); + } + else { + throw new IllegalStateException( + "Only KTable method invocations are permitted"); + } + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderHealthIndicator.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderHealthIndicator.java new file mode 100644 index 000000000..6db6a64ce --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderHealthIndicator.java @@ -0,0 +1,228 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.ListTopicsResult; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.TaskMetadata; +import org.apache.kafka.streams.ThreadMetadata; + +import org.springframework.beans.factory.DisposableBean; +import org.springframework.boot.actuate.health.AbstractHealthIndicator; +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.Status; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; + +/** + * Health indicator for Kafka Streams. + * + * @author Arnaud Jardiné + * @author Soby Chacko + */ +public class KafkaStreamsBinderHealthIndicator extends AbstractHealthIndicator implements DisposableBean { + + /** + * Static initialization for detecting whether the application is using Kafka client 2.5 vs lower versions. + */ + private static ClassLoader CLASS_LOADER = KafkaStreamsBinderHealthIndicator.class.getClassLoader(); + private static boolean isKafkaStreams25 = true; + private static Method methodForIsRunning; + + static { + try { + Class KAFKA_STREAMS_STATE_CLASS = CLASS_LOADER.loadClass("org.apache.kafka.streams.KafkaStreams$State"); + + Method[] declaredMethods = KAFKA_STREAMS_STATE_CLASS.getDeclaredMethods(); + for (Method m : declaredMethods) { + if (m.getName().equals("isRunning")) { + isKafkaStreams25 = false; + methodForIsRunning = m; + } + } + } + catch (ClassNotFoundException e) { + throw new IllegalStateException("KafkaStreams$State class not found", e); + } + } + + private final KafkaStreamsRegistry kafkaStreamsRegistry; + + private final KafkaStreamsBinderConfigurationProperties configurationProperties; + + private final Map adminClientProperties; + + private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; + + private AdminClient adminClient; + + private final Lock lock = new ReentrantLock(); + + KafkaStreamsBinderHealthIndicator(KafkaStreamsRegistry kafkaStreamsRegistry, + KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties, + KafkaProperties kafkaProperties, + KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue) { + super("Kafka-streams health check failed"); + kafkaProperties.buildAdminProperties(); + this.configurationProperties = kafkaStreamsBinderConfigurationProperties; + this.adminClientProperties = kafkaProperties.buildAdminProperties(); + KafkaTopicProvisioner.normalalizeBootPropsWithBinder(this.adminClientProperties, kafkaProperties, + kafkaStreamsBinderConfigurationProperties); + this.kafkaStreamsRegistry = kafkaStreamsRegistry; + this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue; + } + + @Override + protected void doHealthCheck(Health.Builder builder) throws Exception { + try { + this.lock.lock(); + if (this.adminClient == null) { + this.adminClient = AdminClient.create(this.adminClientProperties); + } + + final ListTopicsResult listTopicsResult = this.adminClient.listTopics(); + listTopicsResult.listings().get(this.configurationProperties.getHealthTimeout(), TimeUnit.SECONDS); + + if (this.kafkaStreamsBindingInformationCatalogue.getStreamsBuilderFactoryBeans().isEmpty()) { + builder.withDetail("No Kafka Streams bindings have been established", "Kafka Streams binder did not detect any processors"); + builder.status(Status.UNKNOWN); + } + else { + boolean up = true; + final Set kafkaStreams = kafkaStreamsRegistry.getKafkaStreams(); + Set allKafkaStreams = new HashSet<>(kafkaStreams); + if (this.configurationProperties.isIncludeStoppedProcessorsForHealthCheck()) { + allKafkaStreams.addAll(kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().values()); + } + for (KafkaStreams kStream : allKafkaStreams) { + if (isKafkaStreams25) { + up &= kStream.state().isRunningOrRebalancing(); + } + else { + // if Kafka client version is lower than 2.5, then call the method reflectively. + final boolean isRunningInvokedResult = (boolean) methodForIsRunning.invoke(kStream.state()); + up &= isRunningInvokedResult; + } + builder.withDetails(buildDetails(kStream)); + } + builder.status(up ? Status.UP : Status.DOWN); + } + } + catch (Exception e) { + builder.withDetail("No topic information available", "Kafka broker is not reachable"); + builder.status(Status.DOWN); + builder.withException(e); + } + finally { + this.lock.unlock(); + } + } + + private Map buildDetails(KafkaStreams kafkaStreams) throws Exception { + final Map details = new HashMap<>(); + final Map perAppdIdDetails = new HashMap<>(); + + boolean isRunningResult; + if (isKafkaStreams25) { + isRunningResult = kafkaStreams.state().isRunningOrRebalancing(); + } + else { + // if Kafka client version is lower than 2.5, then call the method reflectively. + isRunningResult = (boolean) methodForIsRunning.invoke(kafkaStreams.state()); + } + + if (isRunningResult) { + final Set threadMetadata = kafkaStreams.metadataForLocalThreads(); + for (ThreadMetadata metadata : threadMetadata) { + perAppdIdDetails.put("threadName", metadata.threadName()); + perAppdIdDetails.put("threadState", metadata.threadState()); + perAppdIdDetails.put("adminClientId", metadata.adminClientId()); + perAppdIdDetails.put("consumerClientId", metadata.consumerClientId()); + perAppdIdDetails.put("restoreConsumerClientId", metadata.restoreConsumerClientId()); + perAppdIdDetails.put("producerClientIds", metadata.producerClientIds()); + perAppdIdDetails.put("activeTasks", taskDetails(metadata.activeTasks())); + perAppdIdDetails.put("standbyTasks", taskDetails(metadata.standbyTasks())); + } + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsRegistry.streamBuilderFactoryBean(kafkaStreams); + final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG); + details.put(applicationId, perAppdIdDetails); + } + else { + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsRegistry.streamBuilderFactoryBean(kafkaStreams); + String applicationId = null; + if (streamsBuilderFactoryBean != null) { + applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG); + } + else { + final Map stoppedKafkaStreamsPerBinding = kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams(); + for (String appId : stoppedKafkaStreamsPerBinding.keySet()) { + if (stoppedKafkaStreamsPerBinding.get(appId).equals(kafkaStreams)) { + applicationId = appId; + } + } + } + details.put(applicationId, String.format("The processor with application.id %s is down. Current state: %s", applicationId, kafkaStreams.state())); + } + return details; + } + + private static Map taskDetails(Set taskMetadata) { + final Map details = new HashMap<>(); + for (TaskMetadata metadata : taskMetadata) { + details.put("taskId", metadata.taskId()); + if (details.containsKey("partitions")) { + @SuppressWarnings("unchecked") + List partitionsInfo = (List) details.get("partitions"); + partitionsInfo.addAll(addPartitionsInfo(metadata)); + } + else { + details.put("partitions", + addPartitionsInfo(metadata)); + } + } + return details; + } + + private static List addPartitionsInfo(TaskMetadata metadata) { + return metadata.topicPartitions().stream().map( + p -> "partition=" + p.partition() + ", topic=" + p.topic()) + .collect(Collectors.toList()); + } + + @Override + public void destroy() throws Exception { + if (adminClient != null) { + adminClient.close(Duration.ofSeconds(0)); + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderHealthIndicatorConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderHealthIndicatorConfiguration.java new file mode 100644 index 000000000..9a00526d9 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderHealthIndicatorConfiguration.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.actuate.autoconfigure.health.ConditionalOnEnabledHealthIndicator; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Configuration class for Kafka-streams binder health indicator beans. + * + * @author Arnaud Jardiné + */ +@Configuration +@ConditionalOnClass(name = "org.springframework.boot.actuate.health.HealthIndicator") +@ConditionalOnEnabledHealthIndicator("binders") +public class KafkaStreamsBinderHealthIndicatorConfiguration { + + @Bean + public KafkaStreamsBinderHealthIndicator kafkaStreamsBinderHealthIndicator( + ObjectProvider kafkaStreamsRegistry, + @Qualifier("binderConfigurationProperties")KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties, + KafkaProperties kafkaProperties, KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue) { + if (kafkaStreamsRegistry.getIfUnique() != null) { + return new KafkaStreamsBinderHealthIndicator(kafkaStreamsRegistry.getIfUnique(), kafkaStreamsBinderConfigurationProperties, + kafkaProperties, kafkaStreamsBindingInformationCatalogue); + } + return null; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderMetrics.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderMetrics.java new file mode 100644 index 000000000..8cdb0bd78 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderMetrics.java @@ -0,0 +1,240 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.ToDoubleFunction; + +import io.micrometer.core.instrument.FunctionCounter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.binder.MeterBinder; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.streams.KafkaStreams; + +import org.springframework.kafka.config.StreamsBuilderFactoryBean; + +/** + * Kafka Streams binder metrics implementation that exports the metrics available + * through {@link KafkaStreams#metrics()} into a micrometer {@link io.micrometer.core.instrument.MeterRegistry}. + * + * Boot 2.2 users need to rely on this class for the metrics instead of direct support from Micrometer. + * Micrometer added Kafka Streams metrics support in 1.4.0 which Boot 2.3 includes. + * Therefore, the users who are on Boot 2.2, need to rely on these metrics. + * For users who are on 2.3 Boot, this class won't be activated (See the configuration for the various + * conditionals used). + * + * For the most part, this class is a copy of the Micrometer Kafka Streams support that was added in version 1.4.0. + * We will keep this class, as long as we support Boot 2.2.x. + * + * @author Soby Chacko + * @since 3.0.0 + */ +public class KafkaStreamsBinderMetrics { + + static final String DEFAULT_VALUE = "unknown"; + + static final String CLIENT_ID_TAG_NAME = "client-id"; + + static final String METRIC_GROUP_APP_INFO = "app-info"; + + static final String VERSION_METRIC_NAME = "version"; + + static final String START_TIME_METRIC_NAME = "start-time-ms"; + + static final String KAFKA_VERSION_TAG_NAME = "kafka-version"; + + static final String METRIC_NAME_PREFIX = "kafka."; + + static final String METRIC_GROUP_METRICS_COUNT = "kafka-metrics-count"; + + private String kafkaVersion = DEFAULT_VALUE; + + private String clientId = DEFAULT_VALUE; + + private final MeterRegistry meterRegistry; + + private MeterBinder meterBinder; + + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + + private volatile Set currentMeters = new HashSet<>(); + + public KafkaStreamsBinderMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + + public void bindTo(Set streamsBuilderFactoryBeans) { + if (this.meterBinder == null) { + this.meterBinder = registry -> { + if (streamsBuilderFactoryBeans != null) { + for (StreamsBuilderFactoryBean streamsBuilderFactoryBean : streamsBuilderFactoryBeans) { + if (streamsBuilderFactoryBean.isRunning()) { + KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams(); + final Map metrics = kafkaStreams.metrics(); + + prepareToBindMetrics(registry, metrics); + checkAndBindMetrics(registry, metrics); + } + } + } + }; + } + this.meterBinder.bindTo(this.meterRegistry); + } + + public void addMetrics(Set streamsBuilderFactoryBeans) { + synchronized (KafkaStreamsBinderMetrics.this) { + this.bindTo(streamsBuilderFactoryBeans); + } + } + + void prepareToBindMetrics(MeterRegistry registry, Map metrics) { + Metric startTime = null; + for (Map.Entry entry : metrics.entrySet()) { + MetricName name = entry.getKey(); + if (clientId.equals(DEFAULT_VALUE) && name.tags().get(CLIENT_ID_TAG_NAME) != null) { + clientId = name.tags().get(CLIENT_ID_TAG_NAME); + } + if (METRIC_GROUP_APP_INFO.equals(name.group())) { + if (VERSION_METRIC_NAME.equals(name.name())) { + kafkaVersion = (String) entry.getValue().metricValue(); + } + else if (START_TIME_METRIC_NAME.equals(name.name())) { + startTime = entry.getValue(); + } + } + } + if (startTime != null) { + bindMeter(registry, startTime, meterName(startTime), meterTags(startTime)); + } + } + + private void bindMeter(MeterRegistry registry, Metric metric, String name, Iterable tags) { + if (name.endsWith("total") || name.endsWith("count")) { + registerCounter(registry, metric, name, tags); + } + else { + registerGauge(registry, metric, name, tags); + } + } + + private void registerCounter(MeterRegistry registry, Metric metric, String name, Iterable tags) { + FunctionCounter.builder(name, metric, toMetricValue()) + .tags(tags) + .description(metric.metricName().description()) + .register(registry); + } + + private ToDoubleFunction toMetricValue() { + return metric -> ((Number) metric.metricValue()).doubleValue(); + } + + private void registerGauge(MeterRegistry registry, Metric metric, String name, Iterable tags) { + Gauge.builder(name, metric, toMetricValue()) + .tags(tags) + .description(metric.metricName().description()) + .register(registry); + } + + private List meterTags(Metric metric) { + return meterTags(metric, false); + } + + private String meterName(Metric metric) { + String name = METRIC_NAME_PREFIX + metric.metricName().group() + "." + metric.metricName().name(); + return name.replaceAll("-metrics", "").replaceAll("-", "."); + } + + private List meterTags(Metric metric, boolean includeCommonTags) { + List tags = new ArrayList<>(); + metric.metricName().tags().forEach((key, value) -> tags.add(Tag.of(key, value))); + tags.add(Tag.of(KAFKA_VERSION_TAG_NAME, kafkaVersion)); + return tags; + } + + private boolean differentClient(List tags) { + for (Tag tag : tags) { + if (tag.getKey().equals(CLIENT_ID_TAG_NAME)) { + if (!clientId.equals(tag.getValue())) { + return true; + } + } + } + return false; + } + + void checkAndBindMetrics(MeterRegistry registry, Map metrics) { + if (!currentMeters.equals(metrics.keySet())) { + currentMeters = new HashSet<>(metrics.keySet()); + metrics.forEach((name, metric) -> { + //Filter out non-numeric values + if (!(metric.metricValue() instanceof Number)) { + return; + } + + //Filter out metrics from groups that include metadata + if (METRIC_GROUP_APP_INFO.equals(name.group())) { + return; + } + if (METRIC_GROUP_METRICS_COUNT.equals(name.group())) { + return; + } + String meterName = meterName(metric); + List meterTagsWithCommonTags = meterTags(metric, true); + //Kafka has metrics with lower number of tags (e.g. with/without topic or partition tag) + //Remove meters with lower number of tags + boolean hasLessTags = false; + for (Meter other : registry.find(meterName).meters()) { + List tags = other.getId().getTags(); + // Only consider meters from the same client before filtering + if (differentClient(tags)) { + break; + } + if (tags.size() < meterTagsWithCommonTags.size()) { + registry.remove(other); + } + // Check if already exists + else if (tags.size() == meterTagsWithCommonTags.size()) { + if (tags.equals(meterTagsWithCommonTags)) { + return; + } + else { + break; + } + } + else { + hasLessTags = true; + } + } + if (hasLessTags) { + return; + } + bindMeter(registry, metric, meterName, meterTags(metric)); + }); + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderSupportAutoConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderSupportAutoConfiguration.java new file mode 100644 index 000000000..17c4d8723 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderSupportAutoConfiguration.java @@ -0,0 +1,493 @@ +/* + * Copyright 2017-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.lang.reflect.Constructor; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +import io.micrometer.core.instrument.MeterRegistry; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.errors.LogAndContinueExceptionHandler; +import org.apache.kafka.streams.errors.LogAndFailExceptionHandler; + +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingClass; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.context.properties.bind.BindResult; +import org.springframework.boot.context.properties.bind.Bindable; +import org.springframework.boot.context.properties.bind.Binder; +import org.springframework.boot.context.properties.bind.PropertySourcesPlaceholdersResolver; +import org.springframework.boot.context.properties.source.ConfigurationPropertySources; +import org.springframework.cloud.stream.binder.BinderConfiguration; +import org.springframework.cloud.stream.binder.kafka.streams.function.FunctionDetectorCondition; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; +import org.springframework.cloud.stream.binder.kafka.streams.serde.CompositeNonNativeSerde; +import org.springframework.cloud.stream.binder.kafka.streams.serde.MessageConverterDelegateSerde; +import org.springframework.cloud.stream.binding.BindingService; +import org.springframework.cloud.stream.binding.StreamListenerResultAdapter; +import org.springframework.cloud.stream.config.BinderProperties; +import org.springframework.cloud.stream.config.BindingServiceConfiguration; +import org.springframework.cloud.stream.config.BindingServiceProperties; +import org.springframework.cloud.stream.function.StreamFunctionProperties; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.Environment; +import org.springframework.core.env.MapPropertySource; +import org.springframework.integration.context.IntegrationContextUtils; +import org.springframework.integration.support.utils.IntegrationUtils; +import org.springframework.kafka.config.KafkaStreamsConfiguration; +import org.springframework.kafka.config.StreamsBuilderFactoryBeanConfigurer; +import org.springframework.kafka.core.CleanupConfig; +import org.springframework.kafka.streams.KafkaStreamsMicrometerListener; +import org.springframework.kafka.streams.RecoveringDeserializationExceptionHandler; +import org.springframework.lang.Nullable; +import org.springframework.messaging.converter.CompositeMessageConverter; +import org.springframework.util.ObjectUtils; +import org.springframework.util.ReflectionUtils; +import org.springframework.util.StringUtils; + +/** + * Kafka Streams binder configuration. + * + * @author Marius Bogoevici + * @author Soby Chacko + * @author Gary Russell + */ +@Configuration +@EnableConfigurationProperties(KafkaStreamsExtendedBindingProperties.class) +@ConditionalOnBean(BindingService.class) +@AutoConfigureAfter(BindingServiceConfiguration.class) +public class KafkaStreamsBinderSupportAutoConfiguration { + + private static final String KSTREAM_BINDER_TYPE = "kstream"; + + private static final String KTABLE_BINDER_TYPE = "ktable"; + + private static final String GLOBALKTABLE_BINDER_TYPE = "globalktable"; + + private static final String CONSUMER_PROPERTIES_PREFIX = "consumer."; + private static final String PRODUCER_PROPERTIES_PREFIX = "producer."; + + @Bean + @ConfigurationProperties(prefix = "spring.cloud.stream.kafka.streams.binder") + public KafkaStreamsBinderConfigurationProperties binderConfigurationProperties( + KafkaProperties kafkaProperties, ConfigurableEnvironment environment, + BindingServiceProperties properties, ConfigurableApplicationContext context) throws Exception { + final Map binderConfigurations = getBinderConfigurations( + properties); + for (Map.Entry entry : binderConfigurations + .entrySet()) { + final BinderConfiguration binderConfiguration = entry.getValue(); + final String binderType = binderConfiguration.getBinderType(); + if (binderType != null && (binderType.equals(KSTREAM_BINDER_TYPE) + || binderType.equals(KTABLE_BINDER_TYPE) + || binderType.equals(GLOBALKTABLE_BINDER_TYPE))) { + Map binderProperties = new HashMap<>(); + this.flatten(null, binderConfiguration.getProperties(), binderProperties); + environment.getPropertySources().addFirst( + new MapPropertySource(entry.getKey() + "-kafkaStreamsBinderEnv", binderProperties)); + + Binder binder = new Binder(ConfigurationPropertySources.get(environment), + new PropertySourcesPlaceholdersResolver(environment), + IntegrationUtils.getConversionService(context.getBeanFactory()), null); + final Constructor kafkaStreamsBinderConfigurationPropertiesConstructor = + ReflectionUtils.accessibleConstructor(KafkaStreamsBinderConfigurationProperties.class, KafkaProperties.class); + final KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties = + BeanUtils.instantiateClass(kafkaStreamsBinderConfigurationPropertiesConstructor, kafkaProperties); + final BindResult bind = binder.bind("spring.cloud.stream.kafka.streams.binder", Bindable.ofInstance(kafkaStreamsBinderConfigurationProperties)); + context.getBeanFactory().registerSingleton( + entry.getKey() + "-KafkaStreamsBinderConfigurationProperties", + bind.get()); + } + } + return new KafkaStreamsBinderConfigurationProperties(kafkaProperties); + } + + // TODO: Lifted from core - good candidate for exposing as a utility method in core. + private static Map getBinderConfigurations( + BindingServiceProperties properties) { + + Map binderConfigurations = new HashMap<>(); + Map declaredBinders = properties.getBinders(); + + for (Map.Entry binderEntry : declaredBinders + .entrySet()) { + BinderProperties binderProperties = binderEntry.getValue(); + binderConfigurations.put(binderEntry.getKey(), + new BinderConfiguration(binderProperties.getType(), + binderProperties.getEnvironment(), + binderProperties.isInheritEnvironment(), + binderProperties.isDefaultCandidate())); + } + return binderConfigurations; + } + + // TODO: Lifted from core - good candidate for exposing as a utility method in core. + @SuppressWarnings("unchecked") + private void flatten(String propertyName, Object value, + Map flattenedProperties) { + if (value instanceof Map) { + ((Map) value).forEach((k, v) -> flatten( + (propertyName != null ? propertyName + "." : "") + k, v, + flattenedProperties)); + } + else { + flattenedProperties.put(propertyName, value.toString()); + } + } + + @Bean + public KafkaStreamsConfiguration kafkaStreamsConfiguration( + @Qualifier("binderConfigurationProperties") KafkaStreamsBinderConfigurationProperties properties, + Environment environment) { + KafkaProperties kafkaProperties = properties.getKafkaProperties(); + Map streamsProperties = kafkaProperties.buildStreamsProperties(); + if (kafkaProperties.getStreams().getApplicationId() == null) { + String applicationName = environment.getProperty("spring.application.name"); + if (applicationName != null) { + streamsProperties.put(StreamsConfig.APPLICATION_ID_CONFIG, + applicationName); + } + } + return new KafkaStreamsConfiguration(streamsProperties); + } + + @Bean("streamConfigGlobalProperties") + public Map streamConfigGlobalProperties( + @Qualifier("binderConfigurationProperties") KafkaStreamsBinderConfigurationProperties configProperties, + KafkaStreamsConfiguration kafkaStreamsConfiguration, ConfigurableEnvironment environment, + SendToDlqAndContinue sendToDlqAndContinue) { + + Properties properties = kafkaStreamsConfiguration.asProperties(); + + String kafkaConnectionString = configProperties.getKafkaConnectionString(); + + if (kafkaConnectionString != null && kafkaConnectionString.equals("localhost:9092")) { + //Making sure that the application indeed set a property. + String kafkaStreamsBinderBroker = environment.getProperty("spring.cloud.stream.kafka.streams.binder.brokers"); + + if (StringUtils.isEmpty(kafkaStreamsBinderBroker)) { + //Kafka Streams binder specific property for brokers is not set by the application. + //See if there is one configured at the kafka binder level. + String kafkaBinderBroker = environment.getProperty("spring.cloud.stream.kafka.binder.brokers"); + if (!StringUtils.isEmpty(kafkaBinderBroker)) { + kafkaConnectionString = kafkaBinderBroker; + configProperties.setBrokers(kafkaConnectionString); + } + } + } + + if (ObjectUtils.isEmpty(properties.get(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG))) { + properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, + kafkaConnectionString); + } + else { + Object bootstrapServerConfig = properties + .get(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG); + if (bootstrapServerConfig instanceof String) { + @SuppressWarnings("unchecked") + String bootStrapServers = (String) properties + .get(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG); + if (bootStrapServers.equals("localhost:9092")) { + properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, + kafkaConnectionString); + } + } + else if (bootstrapServerConfig instanceof List) { + List bootStrapCollection = (List) bootstrapServerConfig; + if (bootStrapCollection.size() == 1 && bootStrapCollection.get(0).equals("localhost:9092")) { + properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, + kafkaConnectionString); + } + } + } + + String binderProvidedApplicationId = configProperties.getApplicationId(); + if (StringUtils.hasText(binderProvidedApplicationId)) { + properties.put(StreamsConfig.APPLICATION_ID_CONFIG, + binderProvidedApplicationId); + } + + properties.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, + Serdes.ByteArraySerde.class.getName()); + properties.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, + Serdes.ByteArraySerde.class.getName()); + + if (configProperties + .getDeserializationExceptionHandler() == DeserializationExceptionHandler.logAndContinue) { + properties.put( + StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, + LogAndContinueExceptionHandler.class); + } + else if (configProperties + .getDeserializationExceptionHandler() == DeserializationExceptionHandler.logAndFail) { + properties.put( + StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, + LogAndFailExceptionHandler.class); + } + else if (configProperties + .getDeserializationExceptionHandler() == DeserializationExceptionHandler.sendToDlq) { + properties.put( + StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, + RecoveringDeserializationExceptionHandler.class); + properties.put(RecoveringDeserializationExceptionHandler.KSTREAM_DESERIALIZATION_RECOVERER, sendToDlqAndContinue); + } + + if (!ObjectUtils.isEmpty(configProperties.getConfiguration())) { + properties.putAll(configProperties.getConfiguration()); + } + + Map mergedConsumerConfig = new HashMap<>(configProperties.mergedConsumerConfiguration()); + //Adding consumer. prefix if they are missing (in order to differentiate them from other property categories such as stream, producer etc.) + addPrefix(properties, mergedConsumerConfig, CONSUMER_PROPERTIES_PREFIX); + + Map mergedProducerConfig = new HashMap<>(configProperties.mergedProducerConfiguration()); + //Adding producer. prefix if they are missing (in order to differentiate them from other property categories such as stream, consumer etc.) + addPrefix(properties, mergedProducerConfig, PRODUCER_PROPERTIES_PREFIX); + + if (!properties.containsKey(StreamsConfig.REPLICATION_FACTOR_CONFIG)) { + properties.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, + (int) configProperties.getReplicationFactor()); + } + return properties.entrySet().stream().collect( + Collectors.toMap((e) -> String.valueOf(e.getKey()), Map.Entry::getValue)); + } + + private void addPrefix(Properties properties, Map mergedConsProdConfig, String prefix) { + Map mergedConfigs = new HashMap<>(); + for (String key : mergedConsProdConfig.keySet()) { + mergedConfigs.put(key.startsWith(prefix) ? key : prefix + key, mergedConsProdConfig.get(key)); + } + if (!ObjectUtils.isEmpty(mergedConfigs)) { + properties.putAll(mergedConfigs); + } + } + + @Bean + public KStreamStreamListenerResultAdapter kstreamStreamListenerResultAdapter() { + return new KStreamStreamListenerResultAdapter(); + } + + @Bean + public KStreamStreamListenerParameterAdapter kstreamStreamListenerParameterAdapter( + KafkaStreamsMessageConversionDelegate kstreamBoundMessageConversionDelegate, + KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue) { + return new KStreamStreamListenerParameterAdapter( + kstreamBoundMessageConversionDelegate, + KafkaStreamsBindingInformationCatalogue); + } + + @Bean + public KafkaStreamsStreamListenerSetupMethodOrchestrator kafkaStreamsStreamListenerSetupMethodOrchestrator( + BindingServiceProperties bindingServiceProperties, + KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties, + KeyValueSerdeResolver keyValueSerdeResolver, + KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, + KStreamStreamListenerParameterAdapter kafkaStreamListenerParameterAdapter, + Collection streamListenerResultAdapters, + ObjectProvider cleanupConfig, + ObjectProvider customizerProvider, ConfigurableEnvironment environment) { + return new KafkaStreamsStreamListenerSetupMethodOrchestrator( + bindingServiceProperties, kafkaStreamsExtendedBindingProperties, + keyValueSerdeResolver, kafkaStreamsBindingInformationCatalogue, + kafkaStreamListenerParameterAdapter, streamListenerResultAdapters, + cleanupConfig.getIfUnique(), customizerProvider.getIfUnique(), environment); + } + + @Bean + public KafkaStreamsMessageConversionDelegate messageConversionDelegate( + @Qualifier(IntegrationContextUtils.ARGUMENT_RESOLVER_MESSAGE_CONVERTER_BEAN_NAME) + CompositeMessageConverter compositeMessageConverter, + SendToDlqAndContinue sendToDlqAndContinue, + KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue, + @Qualifier("binderConfigurationProperties") KafkaStreamsBinderConfigurationProperties binderConfigurationProperties) { + return new KafkaStreamsMessageConversionDelegate(compositeMessageConverter, sendToDlqAndContinue, + KafkaStreamsBindingInformationCatalogue, binderConfigurationProperties); + } + + @Bean + public MessageConverterDelegateSerde messageConverterDelegateSerde( + @Qualifier(IntegrationContextUtils.ARGUMENT_RESOLVER_MESSAGE_CONVERTER_BEAN_NAME) + CompositeMessageConverter compositeMessageConverterFactory) { + return new MessageConverterDelegateSerde(compositeMessageConverterFactory); + } + + @Bean + public CompositeNonNativeSerde compositeNonNativeSerde( + @Qualifier(IntegrationContextUtils.ARGUMENT_RESOLVER_MESSAGE_CONVERTER_BEAN_NAME) + CompositeMessageConverter compositeMessageConverterFactory) { + return new CompositeNonNativeSerde(compositeMessageConverterFactory); + } + + @Bean + public KStreamBoundElementFactory kStreamBoundElementFactory( + BindingServiceProperties bindingServiceProperties, + KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue, + EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler) { + return new KStreamBoundElementFactory(bindingServiceProperties, + KafkaStreamsBindingInformationCatalogue, encodingDecodingBindAdviceHandler); + } + + @Bean + public KTableBoundElementFactory kTableBoundElementFactory( + BindingServiceProperties bindingServiceProperties, EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler, + KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue) { + return new KTableBoundElementFactory(bindingServiceProperties, encodingDecodingBindAdviceHandler, KafkaStreamsBindingInformationCatalogue); + } + + @Bean + public GlobalKTableBoundElementFactory globalKTableBoundElementFactory( + BindingServiceProperties properties, EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler, + KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue) { + return new GlobalKTableBoundElementFactory(properties, encodingDecodingBindAdviceHandler, KafkaStreamsBindingInformationCatalogue); + } + + @Bean + public SendToDlqAndContinue sendToDlqAndContinue() { + return new SendToDlqAndContinue(); + } + + @Bean + public KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue() { + return new KafkaStreamsBindingInformationCatalogue(); + } + + @Bean + @SuppressWarnings("unchecked") + @ConditionalOnMissingBean + public KeyValueSerdeResolver keyValueSerdeResolver( + @Qualifier("streamConfigGlobalProperties") Object streamConfigGlobalProperties, + @Qualifier("binderConfigurationProperties")KafkaStreamsBinderConfigurationProperties properties) { + return new KeyValueSerdeResolver( + (Map) streamConfigGlobalProperties, properties); + } + + @Bean + public InteractiveQueryService interactiveQueryServices( + KafkaStreamsRegistry kafkaStreamsRegistry, + @Qualifier("binderConfigurationProperties")KafkaStreamsBinderConfigurationProperties properties) { + return new InteractiveQueryService(kafkaStreamsRegistry, properties); + } + + @Bean + public KafkaStreamsRegistry kafkaStreamsRegistry() { + return new KafkaStreamsRegistry(); + } + + @Bean + public StreamsBuilderFactoryManager streamsBuilderFactoryManager( + KafkaStreamsBindingInformationCatalogue catalogue, + KafkaStreamsRegistry kafkaStreamsRegistry, + @Nullable KafkaStreamsBinderMetrics kafkaStreamsBinderMetrics, + @Nullable KafkaStreamsMicrometerListener listener, KafkaProperties kafkaProperties) { + return new StreamsBuilderFactoryManager(catalogue, kafkaStreamsRegistry, kafkaStreamsBinderMetrics, listener, kafkaProperties); + } + + @Bean + @Conditional(FunctionDetectorCondition.class) + public KafkaStreamsFunctionProcessor kafkaStreamsFunctionProcessor(BindingServiceProperties bindingServiceProperties, + KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties, + KeyValueSerdeResolver keyValueSerdeResolver, + KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, + KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate, + ObjectProvider cleanupConfig, + StreamFunctionProperties streamFunctionProperties, + @Qualifier("binderConfigurationProperties") KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties, + ObjectProvider customizerProvider, ConfigurableEnvironment environment) { + return new KafkaStreamsFunctionProcessor(bindingServiceProperties, kafkaStreamsExtendedBindingProperties, + keyValueSerdeResolver, kafkaStreamsBindingInformationCatalogue, kafkaStreamsMessageConversionDelegate, + cleanupConfig.getIfUnique(), streamFunctionProperties, kafkaStreamsBinderConfigurationProperties, + customizerProvider.getIfUnique(), environment); + } + + @Bean + public EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler() { + return new EncodingDecodingBindAdviceHandler(); + } + + @Configuration + @ConditionalOnMissingBean(value = KafkaStreamsBinderMetrics.class, name = "outerContext") + @ConditionalOnClass(name = "io.micrometer.core.instrument.MeterRegistry") + protected class KafkaStreamsBinderMetricsConfiguration { + + @Bean + @ConditionalOnBean(MeterRegistry.class) + @ConditionalOnMissingBean(KafkaStreamsBinderMetrics.class) + @ConditionalOnMissingClass("org.springframework.kafka.core.MicrometerConsumerListener") + public KafkaStreamsBinderMetrics kafkaStreamsBinderMetrics(MeterRegistry meterRegistry) { + + return new KafkaStreamsBinderMetrics(meterRegistry); + } + + @ConditionalOnClass(name = "org.springframework.kafka.core.MicrometerConsumerListener") + @ConditionalOnBean(MeterRegistry.class) + protected class KafkaMicrometer { + + @Bean + @ConditionalOnMissingBean(name = "binderStreamsListener") + public KafkaStreamsMicrometerListener binderStreamsListener(MeterRegistry meterRegistry) { + return new KafkaStreamsMicrometerListener(meterRegistry); + } + } + } + + @Configuration + @ConditionalOnBean(name = "outerContext") + @ConditionalOnMissingBean(KafkaStreamsBinderMetrics.class) + @ConditionalOnClass(name = "io.micrometer.core.instrument.MeterRegistry") + protected class KafkaStreamsBinderMetricsConfigurationWithMultiBinder { + + @Bean + @ConditionalOnMissingClass("org.springframework.kafka.core.MicrometerConsumerListener") + public KafkaStreamsBinderMetrics kafkaStreamsBinderMetrics(ConfigurableApplicationContext context) { + + MeterRegistry meterRegistry = context.getBean("outerContext", ApplicationContext.class) + .getBean(MeterRegistry.class); + return new KafkaStreamsBinderMetrics(meterRegistry); + } + + @ConditionalOnClass(name = "org.springframework.kafka.core.MicrometerConsumerListener") + @ConditionalOnBean(MeterRegistry.class) + protected class KafkaMicrometer { + + @Bean + @ConditionalOnMissingBean(name = "binderStreamsListener") + public KafkaStreamsMicrometerListener binderStreamsListener(MeterRegistry meterRegistry) { + return new KafkaStreamsMicrometerListener(meterRegistry); + } + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderUtils.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderUtils.java new file mode 100644 index 000000000..947325812 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderUtils.java @@ -0,0 +1,228 @@ +/* + * Copyright 2018-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.streams.kstream.KStream; + +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.utils.DlqDestinationResolver; +import org.springframework.cloud.stream.binder.kafka.utils.DlqPartitionFunction; +import org.springframework.context.ApplicationContext; +import org.springframework.core.MethodParameter; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaOperations; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.DeadLetterPublishingRecoverer; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ObjectUtils; +import org.springframework.util.StringUtils; + +/** + * Common methods used by various Kafka Streams types across the binders. + * + * @author Soby Chacko + * @author Gary Russell + */ +final class KafkaStreamsBinderUtils { + + private static final Log LOGGER = LogFactory.getLog(KafkaStreamsBinderUtils.class); + + private KafkaStreamsBinderUtils() { + + } + + static void prepareConsumerBinding(String name, String group, + ApplicationContext context, KafkaTopicProvisioner kafkaTopicProvisioner, + KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, + ExtendedConsumerProperties properties, + RetryTemplate retryTemplate, + ConfigurableListableBeanFactory beanFactory, String bindingName, + KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, + StreamsBuilderFactoryBean streamsBuilderFactoryBean) { + + ExtendedConsumerProperties extendedConsumerProperties = + (ExtendedConsumerProperties) properties; + + if (binderConfigurationProperties + .getDeserializationExceptionHandler() == DeserializationExceptionHandler.sendToDlq) { + extendedConsumerProperties.getExtension().setEnableDlq(true); + } + // check for deserialization handler at the consumer binding, as that takes precedence. + final DeserializationExceptionHandler deserializationExceptionHandler = + properties.getExtension().getDeserializationExceptionHandler(); + if (deserializationExceptionHandler == DeserializationExceptionHandler.sendToDlq) { + extendedConsumerProperties.getExtension().setEnableDlq(true); + } + + String[] inputTopics = StringUtils.commaDelimitedListToStringArray(name); + for (String inputTopic : inputTopics) { + kafkaTopicProvisioner.provisionConsumerDestination(inputTopic, group, + extendedConsumerProperties); + } + + if (extendedConsumerProperties.getExtension().isEnableDlq()) { + + Map partitionFunctions = + context.getBeansOfType(DlqPartitionFunction.class, false, false); + boolean oneFunctionPresent = partitionFunctions.size() == 1; + Integer dlqPartitions = extendedConsumerProperties.getExtension().getDlqPartitions(); + DlqPartitionFunction partitionFunction = oneFunctionPresent + ? partitionFunctions.values().iterator().next() + : DlqPartitionFunction.determineFallbackFunction(dlqPartitions, LOGGER); + + ProducerFactory producerFactory = getProducerFactory( + new ExtendedProducerProperties<>( + extendedConsumerProperties.getExtension().getDlqProducerProperties()), + binderConfigurationProperties); + kafkaStreamsBindingInformationCatalogue.addDlqProducerFactory(streamsBuilderFactoryBean, producerFactory); + + KafkaOperations kafkaTemplate = new KafkaTemplate<>(producerFactory); + + Map dlqDestinationResolvers = + context.getBeansOfType(DlqDestinationResolver.class, false, false); + + BiFunction, Exception, TopicPartition> destinationResolver = + dlqDestinationResolvers.isEmpty() ? (cr, e) -> new TopicPartition(extendedConsumerProperties.getExtension().getDlqName(), + partitionFunction.apply(group, cr, e)) : + (cr, e) -> new TopicPartition(dlqDestinationResolvers.values().iterator().next().apply(cr, e), + partitionFunction.apply(group, cr, e)); + + DeadLetterPublishingRecoverer kafkaStreamsBinderDlqRecoverer = !dlqDestinationResolvers.isEmpty() || !StringUtils + .isEmpty(extendedConsumerProperties.getExtension().getDlqName()) + ? new DeadLetterPublishingRecoverer(kafkaTemplate, destinationResolver) + : null; + for (String inputTopic : inputTopics) { + if (StringUtils.isEmpty( + extendedConsumerProperties.getExtension().getDlqName()) && dlqDestinationResolvers.isEmpty()) { + destinationResolver = (cr, e) -> new TopicPartition("error." + inputTopic + "." + group, + partitionFunction.apply(group, cr, e)); + kafkaStreamsBinderDlqRecoverer = new DeadLetterPublishingRecoverer(kafkaTemplate, + destinationResolver); + } + SendToDlqAndContinue sendToDlqAndContinue = context + .getBean(SendToDlqAndContinue.class); + sendToDlqAndContinue.addKStreamDlqDispatch(inputTopic, + kafkaStreamsBinderDlqRecoverer); + } + } + + if (!StringUtils.hasText(properties.getRetryTemplateName())) { + @SuppressWarnings("unchecked") + BeanDefinition retryTemplateBeanDefinition = BeanDefinitionBuilder + .genericBeanDefinition( + (Class) retryTemplate.getClass(), + () -> retryTemplate) + .getRawBeanDefinition(); + ((BeanDefinitionRegistry) beanFactory).registerBeanDefinition(bindingName + "-RetryTemplate", retryTemplateBeanDefinition); + } + } + + private static DefaultKafkaProducerFactory getProducerFactory( + ExtendedProducerProperties producerProperties, + KafkaBinderConfigurationProperties configurationProperties) { + Map props = new HashMap<>(); + props.put(ProducerConfig.RETRIES_CONFIG, 0); + props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432); + props.put(ProducerConfig.ACKS_CONFIG, configurationProperties.getRequiredAcks()); + Map mergedConfig = configurationProperties + .mergedProducerConfiguration(); + if (!ObjectUtils.isEmpty(mergedConfig)) { + props.putAll(mergedConfig); + } + if (ObjectUtils.isEmpty(props.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG))) { + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + configurationProperties.getKafkaConnectionString()); + } + if (ObjectUtils.isEmpty(props.get(ProducerConfig.BATCH_SIZE_CONFIG))) { + props.put(ProducerConfig.BATCH_SIZE_CONFIG, + String.valueOf(producerProperties.getExtension().getBufferSize())); + } + if (ObjectUtils.isEmpty(props.get(ProducerConfig.LINGER_MS_CONFIG))) { + props.put(ProducerConfig.LINGER_MS_CONFIG, + String.valueOf(producerProperties.getExtension().getBatchTimeout())); + } + if (ObjectUtils.isEmpty(props.get(ProducerConfig.COMPRESSION_TYPE_CONFIG))) { + props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, + producerProperties.getExtension().getCompressionType().toString()); + } + Map configs = producerProperties.getExtension().getConfiguration(); + Assert.state(!configs.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG), + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " cannot be overridden at the binding level; " + + "use multiple binders instead"); + if (!ObjectUtils.isEmpty(configs)) { + props.putAll(configs); + } + // Always send as byte[] on dlq (the same byte[] that the consumer received) + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + ByteArraySerializer.class); + + return new DefaultKafkaProducerFactory<>(props); + } + + + static boolean supportsKStream(MethodParameter methodParameter, Class targetBeanClass) { + return KStream.class.isAssignableFrom(targetBeanClass) + && KStream.class.isAssignableFrom(methodParameter.getParameterType()); + } + + static void closeDlqProducerFactories(KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, + StreamsBuilderFactoryBean streamsBuilderFactoryBean) { + + final List> dlqProducerFactories = + kafkaStreamsBindingInformationCatalogue.getDlqProducerFactory(streamsBuilderFactoryBean); + + if (!CollectionUtils.isEmpty(dlqProducerFactories)) { + for (ProducerFactory producerFactory : dlqProducerFactories) { + try { + ((DisposableBean) producerFactory).destroy(); + } + catch (Exception exception) { + throw new IllegalStateException(exception); + } + } + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java new file mode 100644 index 000000000..366da7e04 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBindingInformationCatalogue.java @@ -0,0 +1,250 @@ +/* + * Copyright 2018-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.KStream; + +import org.springframework.cloud.stream.binder.ConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; +import org.springframework.cloud.stream.config.BindingProperties; +import org.springframework.core.ResolvableType; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.util.CollectionUtils; + +/** + * A catalogue that provides binding information for Kafka Streams target types such as + * KStream. It also keeps a catalogue for the underlying {@link StreamsBuilderFactoryBean} + * and {@link StreamsConfig} associated with various + * {@link org.springframework.cloud.stream.annotation.StreamListener} methods in the + * {@link org.springframework.context.ApplicationContext}. + * + * @author Soby Chacko + */ +public class KafkaStreamsBindingInformationCatalogue { + + private final Map, BindingProperties> bindingProperties = new ConcurrentHashMap<>(); + + private final Map, KafkaStreamsConsumerProperties> consumerProperties = new ConcurrentHashMap<>(); + + private final Map streamsBuilderFactoryBeanPerBinding = new HashMap<>(); + + private final Map> consumerPropertiesPerSbfb = new HashMap<>(); + + private final Map outboundKStreamResolvables = new HashMap<>(); + + private final Map, Serde> keySerdeInfo = new HashMap<>(); + + private final Map bindingNamesPerTarget = new HashMap<>(); + + private final Map previousKafkaStreamsPerApplicationId = new HashMap<>(); + + private final Map>> dlqProducerFactories = new HashMap<>(); + + /** + * For a given bounded {@link KStream}, retrieve it's corresponding destination on the + * broker. + * @param bindingTarget binding target for KStream + * @return destination topic on Kafka + */ + String getDestination(KStream bindingTarget) { + BindingProperties bindingProperties = this.bindingProperties.get(bindingTarget); + return bindingProperties.getDestination(); + } + + /** + * Is native decoding is enabled on this {@link KStream}. + * @param bindingTarget binding target for KStream + * @return true if native decoding is enabled, fasle otherwise. + */ + boolean isUseNativeDecoding(KStream bindingTarget) { + BindingProperties bindingProperties = this.bindingProperties.get(bindingTarget); + if (bindingProperties.getConsumer() == null) { + bindingProperties.setConsumer(new ConsumerProperties()); + } + return bindingProperties.getConsumer().isUseNativeDecoding(); + } + + /** + * Is DLQ enabled for this {@link KStream}. + * @param bindingTarget binding target for KStream + * @return true if DLQ is enabled, false otherwise. + */ + boolean isDlqEnabled(KStream bindingTarget) { + return this.consumerProperties.get(bindingTarget).isEnableDlq(); + } + + /** + * Retrieve the content type associated with a given {@link KStream}. + * @param bindingTarget binding target for KStream + * @return content Type associated. + */ + String getContentType(KStream bindingTarget) { + BindingProperties bindingProperties = this.bindingProperties.get(bindingTarget); + return bindingProperties.getContentType(); + } + + /** + * Register a cache for bounded KStream -> {@link BindingProperties}. + * @param bindingTarget binding target for KStream + * @param bindingProperties {@link BindingProperties} for this KStream + */ + void registerBindingProperties(KStream bindingTarget, + BindingProperties bindingProperties) { + if (bindingProperties != null) { + this.bindingProperties.put(bindingTarget, bindingProperties); + } + } + + /** + * Register a cache for bounded KStream -> {@link KafkaStreamsConsumerProperties}. + * @param bindingTarget binding target for KStream + * @param kafkaStreamsConsumerProperties consumer properties for this KStream + */ + void registerConsumerProperties(KStream bindingTarget, + KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties) { + if (kafkaStreamsConsumerProperties != null) { + this.consumerProperties.put(bindingTarget, kafkaStreamsConsumerProperties); + } + } + + Set getStreamsBuilderFactoryBeans() { + return new HashSet<>(this.streamsBuilderFactoryBeanPerBinding.values()); + } + + void addStreamBuilderFactoryPerBinding(String binding, StreamsBuilderFactoryBean streamsBuilderFactoryBean) { + this.streamsBuilderFactoryBeanPerBinding.put(binding, streamsBuilderFactoryBean); + } + + void addConsumerPropertiesPerSbfb(StreamsBuilderFactoryBean streamsBuilderFactoryBean, ConsumerProperties consumerProperties) { + this.consumerPropertiesPerSbfb.computeIfAbsent(streamsBuilderFactoryBean, k -> new ArrayList<>()); + this.consumerPropertiesPerSbfb.get(streamsBuilderFactoryBean).add(consumerProperties); + } + + public Map> getConsumerPropertiesPerSbfb() { + return this.consumerPropertiesPerSbfb; + } + + Map getStreamsBuilderFactoryBeanPerBinding() { + return this.streamsBuilderFactoryBeanPerBinding; + } + + void addOutboundKStreamResolvable(Object key, ResolvableType outboundResolvable) { + this.outboundKStreamResolvables.put(key, outboundResolvable); + } + + ResolvableType getOutboundKStreamResolvable(Object key) { + return outboundKStreamResolvables.get(key); + } + + /** + * Adding a mapping for KStream target to its corresponding KeySerde. + * This is used for sending to DLQ when deserialization fails. See {@link KafkaStreamsMessageConversionDelegate} + * for details. + * + * @param kStreamTarget target KStream + * @param keySerde Serde used for the key + */ + void addKeySerde(KStream kStreamTarget, Serde keySerde) { + this.keySerdeInfo.put(kStreamTarget, keySerde); + } + + Serde getKeySerde(KStream kStreamTarget) { + return this.keySerdeInfo.get(kStreamTarget); + } + + + Map, BindingProperties> getBindingProperties() { + return bindingProperties; + } + + Map, KafkaStreamsConsumerProperties> getConsumerProperties() { + return consumerProperties; + } + + void addBindingNamePerTarget(Object target, String bindingName) { + this.bindingNamesPerTarget.put(target, bindingName); + } + + String bindingNamePerTarget(Object target) { + return this.bindingNamesPerTarget.get(target); + } + + public List> getDlqProducerFactories() { + return this.dlqProducerFactories.values() + .stream() + .flatMap(List::stream) + .collect(Collectors.toList()); + } + + public List> getDlqProducerFactory(StreamsBuilderFactoryBean streamsBuilderFactoryBean) { + return this.dlqProducerFactories.get(streamsBuilderFactoryBean); + } + + public void addDlqProducerFactory(StreamsBuilderFactoryBean streamsBuilderFactoryBean, + ProducerFactory producerFactory) { + List> producerFactories = this.dlqProducerFactories.get(streamsBuilderFactoryBean); + if (CollectionUtils.isEmpty(producerFactories)) { + producerFactories = new ArrayList<>(); + this.dlqProducerFactories.put(streamsBuilderFactoryBean, producerFactories); + } + producerFactories.add(producerFactory); + } + + /** + * Caching the previous KafkaStreams for the applicaiton.id when binding is stopped through actuator. + * See https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + * + * @param applicationId application.id + * @param kafkaStreams {@link KafkaStreams} object + */ + public void addPreviousKafkaStreamsForApplicationId(String applicationId, KafkaStreams kafkaStreams) { + this.previousKafkaStreamsPerApplicationId.put(applicationId, kafkaStreams); + } + + /** + * Remove the previously cached KafkaStreams object. + * See https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + * + * @param applicationId application.id + */ + public void removePreviousKafkaStreamsForApplicationId(String applicationId) { + this.previousKafkaStreamsPerApplicationId.remove(applicationId); + } + + /** + * Get all stopped KafkaStreams objects through actuator binding stop. + * See https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165 + * + * @return stopped KafkaStreams objects map + */ + public Map getStoppedKafkaStreams() { + return this.previousKafkaStreamsPerApplicationId; + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionProcessor.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionProcessor.java new file mode 100644 index 000000000..c91953715 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionProcessor.java @@ -0,0 +1,575 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.Topology; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.BeanFactoryAware; +import org.springframework.beans.factory.BeanInitializationException; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.beans.factory.support.RootBeanDefinition; +import org.springframework.cloud.stream.binder.kafka.streams.function.KafkaStreamsBindableProxyFactory; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; +import org.springframework.cloud.stream.binding.StreamListenerErrorMessages; +import org.springframework.cloud.stream.config.BindingProperties; +import org.springframework.cloud.stream.config.BindingServiceProperties; +import org.springframework.cloud.stream.function.FunctionConstants; +import org.springframework.cloud.stream.function.StreamFunctionProperties; +import org.springframework.core.ResolvableType; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.config.StreamsBuilderFactoryBeanConfigurer; +import org.springframework.kafka.core.CleanupConfig; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +/** + * @author Soby Chacko + * @since 2.2.0 + */ +public class KafkaStreamsFunctionProcessor extends AbstractKafkaStreamsBinderProcessor implements BeanFactoryAware { + + private static final Log LOG = LogFactory.getLog(KafkaStreamsFunctionProcessor.class); + private static final String OUTBOUND = "outbound"; + + private final BindingServiceProperties bindingServiceProperties; + private final Map methodStreamsBuilderFactoryBeanMap = new HashMap<>(); + private final KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties; + private final KeyValueSerdeResolver keyValueSerdeResolver; + private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; + private final KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate; + + private BeanFactory beanFactory; + private StreamFunctionProperties streamFunctionProperties; + private KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties; + StreamsBuilderFactoryBeanConfigurer customizer; + ConfigurableEnvironment environment; + + public KafkaStreamsFunctionProcessor(BindingServiceProperties bindingServiceProperties, + KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties, + KeyValueSerdeResolver keyValueSerdeResolver, + KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, + KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate, + CleanupConfig cleanupConfig, + StreamFunctionProperties streamFunctionProperties, + KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties, + StreamsBuilderFactoryBeanConfigurer customizer, ConfigurableEnvironment environment) { + super(bindingServiceProperties, kafkaStreamsBindingInformationCatalogue, kafkaStreamsExtendedBindingProperties, + keyValueSerdeResolver, cleanupConfig); + this.bindingServiceProperties = bindingServiceProperties; + this.kafkaStreamsExtendedBindingProperties = kafkaStreamsExtendedBindingProperties; + this.keyValueSerdeResolver = keyValueSerdeResolver; + this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue; + this.kafkaStreamsMessageConversionDelegate = kafkaStreamsMessageConversionDelegate; + this.streamFunctionProperties = streamFunctionProperties; + this.kafkaStreamsBinderConfigurationProperties = kafkaStreamsBinderConfigurationProperties; + this.customizer = customizer; + this.environment = environment; + } + + private Map buildTypeMap(ResolvableType resolvableType, + KafkaStreamsBindableProxyFactory kafkaStreamsBindableProxyFactory, + Method method, String functionName) { + Map resolvableTypeMap = new LinkedHashMap<>(); + if (method != null) { // Component functional bean. + final ResolvableType firstMethodParameter = ResolvableType.forMethodParameter(method, 0); + ResolvableType currentOutputGeneric = ResolvableType.forMethodReturnType(method); + + final Set inputs = new LinkedHashSet<>(kafkaStreamsBindableProxyFactory.getInputs()); + final Iterator iterator = inputs.iterator(); + populateResolvableTypeMap(firstMethodParameter, resolvableTypeMap, iterator, method, functionName); + + final Class outputRawclass = currentOutputGeneric.getRawClass(); + traverseReturnTypeForComponentBeans(resolvableTypeMap, currentOutputGeneric, inputs, iterator, outputRawclass); + } + else if (resolvableType != null && resolvableType.getRawClass() != null) { + int inputCount = 1; + + ResolvableType currentOutputGeneric; + if (resolvableType.getRawClass().isAssignableFrom(BiFunction.class) || + resolvableType.getRawClass().isAssignableFrom(BiConsumer.class)) { + inputCount = 2; + currentOutputGeneric = resolvableType.getGeneric(2); + } + else { + currentOutputGeneric = resolvableType.getGeneric(1); + } + while (currentOutputGeneric.getRawClass() != null && functionOrConsumerFound(currentOutputGeneric)) { + inputCount++; + currentOutputGeneric = currentOutputGeneric.getGeneric(1); + } + final Set inputs = new LinkedHashSet<>(kafkaStreamsBindableProxyFactory.getInputs()); + + final Iterator iterator = inputs.iterator(); + + populateResolvableTypeMap(resolvableType, resolvableTypeMap, iterator); + + ResolvableType iterableResType = resolvableType; + int i = resolvableType.getRawClass().isAssignableFrom(BiFunction.class) || + resolvableType.getRawClass().isAssignableFrom(BiConsumer.class) ? 2 : 1; + ResolvableType outboundResolvableType; + if (i == inputCount) { + outboundResolvableType = iterableResType.getGeneric(i); + } + else { + while (i < inputCount && iterator.hasNext()) { + iterableResType = iterableResType.getGeneric(1); + if (iterableResType.getRawClass() != null && + functionOrConsumerFound(iterableResType)) { + populateResolvableTypeMap(iterableResType, resolvableTypeMap, iterator); + } + i++; + } + outboundResolvableType = iterableResType.getGeneric(1); + } + resolvableTypeMap.put(OUTBOUND, outboundResolvableType); + } + return resolvableTypeMap; + } + + private void traverseReturnTypeForComponentBeans(Map resolvableTypeMap, ResolvableType currentOutputGeneric, + Set inputs, Iterator iterator, Class outputRawclass) { + if (outputRawclass != null && !outputRawclass.equals(Void.TYPE)) { + ResolvableType iterableResType = currentOutputGeneric; + int i = 1; + // Traverse through the return signature. + while (i < inputs.size() && iterator.hasNext()) { + if (iterableResType.getRawClass() != null && + functionOrConsumerFound(iterableResType)) { + populateResolvableTypeMap(iterableResType, resolvableTypeMap, iterator); + } + iterableResType = iterableResType.getGeneric(1); + i++; + } + if (iterableResType.getRawClass() != null && KStream.class.isAssignableFrom(iterableResType.getRawClass())) { + resolvableTypeMap.put(OUTBOUND, iterableResType); + } + } + } + + private boolean functionOrConsumerFound(ResolvableType iterableResType) { + return iterableResType.getRawClass().equals(Function.class) || + iterableResType.getRawClass().equals(Consumer.class); + } + + private void populateResolvableTypeMap(ResolvableType resolvableType, Map resolvableTypeMap, + Iterator iterator) { + final String next = iterator.next(); + resolvableTypeMap.put(next, resolvableType.getGeneric(0)); + if (resolvableType.getRawClass() != null && + (resolvableType.getRawClass().isAssignableFrom(BiFunction.class) || + resolvableType.getRawClass().isAssignableFrom(BiConsumer.class)) + && iterator.hasNext()) { + resolvableTypeMap.put(iterator.next(), resolvableType.getGeneric(1)); + } + } + + private void populateResolvableTypeMap(ResolvableType resolvableType, Map resolvableTypeMap, + Iterator iterator, Method method, String functionName) { + final String next = iterator.next(); + resolvableTypeMap.put(next, resolvableType); + if (method != null) { + final Object bean = beanFactory.getBean(functionName); + if (BiFunction.class.isAssignableFrom(bean.getClass()) || BiConsumer.class.isAssignableFrom(bean.getClass())) { + resolvableTypeMap.put(iterator.next(), ResolvableType.forMethodParameter(method, 1)); + } + } + } + + private ResolvableType checkOutboundForComposedFunctions( + ResolvableType outputResolvableType) { + + ResolvableType currentOutputGeneric; + + if (outputResolvableType.getRawClass() != null && outputResolvableType.getRawClass().isAssignableFrom(BiFunction.class)) { + currentOutputGeneric = outputResolvableType.getGeneric(2); + } + else { + currentOutputGeneric = outputResolvableType.getGeneric(1); + } + while (currentOutputGeneric.getRawClass() != null && functionOrConsumerFound(currentOutputGeneric)) { + currentOutputGeneric = currentOutputGeneric.getGeneric(1); + } + return currentOutputGeneric; + } + + /** + * This method must be kept stateless. In the case of multiple function beans in an application, + * isolated {@link KafkaStreamsBindableProxyFactory} instances are passed in separately for those functions. If the + * state is shared between invocations, that will create potential race conditions. Hence, invocations of this method + * should not be dependent on state modified by a previous invocation. + * + * @param resolvableType type of the binding + * @param functionName bean name of the function + * @param kafkaStreamsBindableProxyFactory bindable proxy factory for the Kafka Streams type + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + public void setupFunctionInvokerForKafkaStreams(ResolvableType resolvableType, String functionName, + KafkaStreamsBindableProxyFactory kafkaStreamsBindableProxyFactory, Method method, + ResolvableType outputResolvableType, + String... composedFunctionNames) { + final Map resolvableTypes = buildTypeMap(resolvableType, + kafkaStreamsBindableProxyFactory, method, functionName); + + ResolvableType outboundResolvableType; + if (outputResolvableType != null) { + outboundResolvableType = checkOutboundForComposedFunctions(outputResolvableType); + resolvableTypes.remove(OUTBOUND); + } + else { + outboundResolvableType = resolvableTypes.remove(OUTBOUND); + } + + Object[] adaptedInboundArguments = adaptAndRetrieveInboundArguments(resolvableTypes, functionName); + try { + if (resolvableType.getRawClass() != null && resolvableType.getRawClass().equals(Consumer.class)) { + Consumer consumer = (Consumer) this.beanFactory.getBean(functionName); + consumer.accept(adaptedInboundArguments[0]); + } + else if (resolvableType.getRawClass() != null && resolvableType.getRawClass().equals(BiConsumer.class)) { + BiConsumer biConsumer = (BiConsumer) this.beanFactory.getBean(functionName); + biConsumer.accept(adaptedInboundArguments[0], adaptedInboundArguments[1]); + } + else if (method != null) { // Handling component functional beans + final Object bean = beanFactory.getBean(functionName); + if (Consumer.class.isAssignableFrom(bean.getClass())) { + ((Consumer) bean).accept(adaptedInboundArguments[0]); + } + else if (BiConsumer.class.isAssignableFrom(bean.getClass())) { + ((BiConsumer) bean).accept(adaptedInboundArguments[0], adaptedInboundArguments[1]); + } + else if (Function.class.isAssignableFrom(bean.getClass()) || BiFunction.class.isAssignableFrom(bean.getClass())) { + Object result; + if (BiFunction.class.isAssignableFrom(bean.getClass())) { + result = ((BiFunction) bean).apply(adaptedInboundArguments[0], adaptedInboundArguments[1]); + } + else { + result = ((Function) bean).apply(adaptedInboundArguments[0]); + } + result = handleCurriedFunctions(adaptedInboundArguments, result); + if (result != null) { + final Set outputs = new TreeSet<>(kafkaStreamsBindableProxyFactory.getOutputs()); + final Iterator outboundDefinitionIterator = outputs.iterator(); + if (result.getClass().isArray()) { + final String initialInput = resolvableTypes.keySet().iterator().next(); + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = + this.kafkaStreamsBindingInformationCatalogue.getStreamsBuilderFactoryBeanPerBinding().get(initialInput); + handleKStreamArrayOutbound(resolvableType, functionName, kafkaStreamsBindableProxyFactory, + outboundResolvableType, (Object[]) result, streamsBuilderFactoryBean); + } + else { + if (KTable.class.isAssignableFrom(result.getClass())) { + handleSingleKStreamOutbound(resolvableTypes, outboundResolvableType != null ? + outboundResolvableType : resolvableType.getGeneric(1), ((KTable) result).toStream(), outboundDefinitionIterator); + } + else { + handleSingleKStreamOutbound(resolvableTypes, outboundResolvableType, (KStream) result, outboundDefinitionIterator); + } + } + } + } + } + else { + Object result = null; + if (resolvableType.getRawClass() != null && resolvableType.getRawClass().equals(BiFunction.class)) { + if (composedFunctionNames != null && composedFunctionNames.length > 0) { + result = handleComposedFunctions(adaptedInboundArguments, result, composedFunctionNames); + } + else { + BiFunction biFunction = (BiFunction) beanFactory.getBean(functionName); + result = biFunction.apply(adaptedInboundArguments[0], adaptedInboundArguments[1]); + result = handleCurriedFunctions(adaptedInboundArguments, result); + } + } + else { + if (composedFunctionNames != null && composedFunctionNames.length > 0) { + result = handleComposedFunctions(adaptedInboundArguments, result, composedFunctionNames); + } + else { + Function function = (Function) beanFactory.getBean(functionName); + result = function.apply(adaptedInboundArguments[0]); + result = handleCurriedFunctions(adaptedInboundArguments, result); + } + } + if (result != null) { + final Set outputs = new TreeSet<>(kafkaStreamsBindableProxyFactory.getOutputs()); + final Iterator outboundDefinitionIterator = outputs.iterator(); + + if (result.getClass().isArray()) { + final String initialInput = resolvableTypes.keySet().iterator().next(); + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = + this.kafkaStreamsBindingInformationCatalogue.getStreamsBuilderFactoryBeanPerBinding().get(initialInput); + handleKStreamArrayOutbound(resolvableType, functionName, kafkaStreamsBindableProxyFactory, + outboundResolvableType, (Object[]) result, streamsBuilderFactoryBean); + } + else { + if (KTable.class.isAssignableFrom(result.getClass())) { + handleSingleKStreamOutbound(resolvableTypes, outboundResolvableType != null ? + outboundResolvableType : resolvableType.getGeneric(1), ((KTable) result).toStream(), outboundDefinitionIterator); + } + else { + handleSingleKStreamOutbound(resolvableTypes, outboundResolvableType != null ? + outboundResolvableType : resolvableType.getGeneric(1), (KStream) result, outboundDefinitionIterator); + } + } + } + } + } + catch (Exception ex) { + throw new BeanInitializationException("Cannot setup function invoker for this Kafka Streams function.", ex); + } + } + + @SuppressWarnings({"unchecked"}) + private Object handleComposedFunctions(Object[] adaptedInboundArguments, Object result, String... composedFunctionNames) { + Object bean = beanFactory.getBean(composedFunctionNames[0]); + if (BiFunction.class.isAssignableFrom(bean.getClass())) { + result = ((BiFunction) bean).apply(adaptedInboundArguments[0], adaptedInboundArguments[1]); + } + else if (Function.class.isAssignableFrom(bean.getClass())) { + result = ((Function) bean).apply(adaptedInboundArguments[0]); + } + // If the return is a curried function, apply it + result = handleCurriedFunctions(adaptedInboundArguments, result); + // Apply composed functions + return applyComposedFunctions(result, composedFunctionNames); + } + + @SuppressWarnings({"unchecked"}) + private Object applyComposedFunctions(Object result, String[] composedFunctionNames) { + for (int i = 1; i < composedFunctionNames.length; i++) { + final Object bean = beanFactory.getBean(composedFunctionNames[i]); + if (Consumer.class.isAssignableFrom(bean.getClass())) { + ((Consumer) bean).accept(result); + result = null; + } + else if (Function.class.isAssignableFrom(bean.getClass())) { + result = ((Function) bean).apply(result); + } + else { + throw new IllegalStateException("You can only compose functions of type either java.util.function.Function or java.util.function.Consumer."); + } + } + return result; + } + + @SuppressWarnings({"unchecked"}) + private Object handleCurriedFunctions(Object[] adaptedInboundArguments, Object result) { + int i = 1; + while (result instanceof Function || result instanceof Consumer) { + if (result instanceof Function) { + result = ((Function) result).apply(adaptedInboundArguments[i]); + } + else { + ((Consumer) result).accept(adaptedInboundArguments[i]); + result = null; + } + i++; + } + return result; + } + + private void handleSingleKStreamOutbound(Map resolvableTypes, ResolvableType outboundResolvableType, + KStream result, Iterator outboundDefinitionIterator) { + if (outboundDefinitionIterator.hasNext()) { + String outbound = outboundDefinitionIterator.next(); + Object targetBean = handleSingleKStreamOutbound(result, outbound); + kafkaStreamsBindingInformationCatalogue.addOutboundKStreamResolvable(targetBean, + outboundResolvableType); + + final String next = resolvableTypes.keySet().iterator().next(); + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsBindingInformationCatalogue + .getStreamsBuilderFactoryBeanPerBinding().get(next); + this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding(outbound, streamsBuilderFactoryBean); + } + } + + private Object handleSingleKStreamOutbound(KStream result, String next) { + Object targetBean = this.applicationContext.getBean(next); + KStreamBoundElementFactory.KStreamWrapper + boundElement = (KStreamBoundElementFactory.KStreamWrapper) targetBean; + boundElement.wrap(result); + return targetBean; + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + private void handleKStreamArrayOutbound(ResolvableType resolvableType, String functionName, + KafkaStreamsBindableProxyFactory kafkaStreamsBindableProxyFactory, + ResolvableType outboundResolvableType, Object[] result, + StreamsBuilderFactoryBean streamsBuilderFactoryBean) { + // Binding target as the output bindings were deferred in the KafkaStreamsBindableProxyFactory + // due to the fact that it didn't know the returned array size. At this point in the execution, + // we know exactly the number of outbound components (from the array length), so do the binding. + final int length = result.length; + + List outputBindings = getOutputBindings(functionName, length); + Iterator iterator = outputBindings.iterator(); + BeanDefinitionRegistry registry = (BeanDefinitionRegistry) beanFactory; + + for (Object o : result) { + String next = iterator.next(); + kafkaStreamsBindableProxyFactory.addOutputBinding(next, KStream.class); + RootBeanDefinition rootBeanDefinition1 = new RootBeanDefinition(); + rootBeanDefinition1.setInstanceSupplier(() -> kafkaStreamsBindableProxyFactory.getOutputHolders().get(next).getBoundTarget()); + registry.registerBeanDefinition(next, rootBeanDefinition1); + + Object targetBean = this.applicationContext.getBean(next); + + KStreamBoundElementFactory.KStreamWrapper + boundElement = (KStreamBoundElementFactory.KStreamWrapper) targetBean; + boundElement.wrap((KStream) o); + + kafkaStreamsBindingInformationCatalogue.addOutboundKStreamResolvable( + targetBean, outboundResolvableType != null ? outboundResolvableType : resolvableType.getGeneric(1)); + + this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding(next, streamsBuilderFactoryBean); + } + } + + private List getOutputBindings(String functionName, int outputs) { + List outputBindings = this.streamFunctionProperties.getOutputBindings(functionName); + List outputBindingNames = new ArrayList<>(); + if (!CollectionUtils.isEmpty(outputBindings)) { + outputBindingNames.addAll(outputBindings); + return outputBindingNames; + } + else { + for (int i = 0; i < outputs; i++) { + outputBindingNames.add(String.format("%s-%s-%d", functionName, FunctionConstants.DEFAULT_OUTPUT_SUFFIX, i)); + } + } + return outputBindingNames; + + } + + @SuppressWarnings({"unchecked"}) + private Object[] adaptAndRetrieveInboundArguments(Map stringResolvableTypeMap, + String functionName) { + Object[] arguments = new Object[stringResolvableTypeMap.size()]; + int i = 0; + for (String input : stringResolvableTypeMap.keySet()) { + Class parameterType = stringResolvableTypeMap.get(input).getRawClass(); + + if (input != null) { + Object targetBean = applicationContext.getBean(input); + BindingProperties bindingProperties = this.bindingServiceProperties.getBindingProperties(input); + //Retrieve the StreamsConfig created for this method if available. + //Otherwise, create the StreamsBuilderFactory and get the underlying config. + if (!this.methodStreamsBuilderFactoryBeanMap.containsKey(functionName)) { + StreamsBuilderFactoryBean streamsBuilderFactoryBean = buildStreamsBuilderAndRetrieveConfig(functionName, applicationContext, + input, kafkaStreamsBinderConfigurationProperties, customizer, this.environment, bindingProperties); + this.methodStreamsBuilderFactoryBeanMap.put(functionName, streamsBuilderFactoryBean); + } + try { + StreamsBuilderFactoryBean streamsBuilderFactoryBean = + this.methodStreamsBuilderFactoryBeanMap.get(functionName); + StreamsBuilder streamsBuilder = streamsBuilderFactoryBean.getObject(); + final String applicationId = streamsBuilderFactoryBean.getStreamsConfiguration().getProperty(StreamsConfig.APPLICATION_ID_CONFIG); + KafkaStreamsConsumerProperties extendedConsumerProperties = + this.kafkaStreamsExtendedBindingProperties.getExtendedConsumerProperties(input); + extendedConsumerProperties.setApplicationId(applicationId); + //get state store spec + + Serde keySerde = this.keyValueSerdeResolver.getInboundKeySerde(extendedConsumerProperties, stringResolvableTypeMap.get(input)); + LOG.info("Key Serde used for " + input + ": " + keySerde.getClass().getName()); + Serde valueSerde = bindingServiceProperties.getConsumerProperties(input).isUseNativeDecoding() ? + getValueSerde(input, extendedConsumerProperties, stringResolvableTypeMap.get(input)) : Serdes.ByteArray(); + LOG.info("Value Serde used for " + input + ": " + valueSerde.getClass().getName()); + final Topology.AutoOffsetReset autoOffsetReset = getAutoOffsetReset(input, extendedConsumerProperties); + + if (parameterType.isAssignableFrom(KStream.class)) { + KStream stream = getKStream(input, bindingProperties, extendedConsumerProperties, + streamsBuilder, keySerde, valueSerde, autoOffsetReset, i == 0); + KStreamBoundElementFactory.KStreamWrapper kStreamWrapper = + (KStreamBoundElementFactory.KStreamWrapper) targetBean; + //wrap the proxy created during the initial target type binding with real object (KStream) + kStreamWrapper.wrap((KStream) stream); + + this.kafkaStreamsBindingInformationCatalogue.addKeySerde((KStream) kStreamWrapper, keySerde); + + this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding(input, streamsBuilderFactoryBean); + this.kafkaStreamsBindingInformationCatalogue.addConsumerPropertiesPerSbfb(streamsBuilderFactoryBean, + bindingServiceProperties.getConsumerProperties(input)); + + if (KStream.class.isAssignableFrom(stringResolvableTypeMap.get(input).getRawClass())) { + final Class valueClass = + (stringResolvableTypeMap.get(input).getGeneric(1).getRawClass() != null) + ? (stringResolvableTypeMap.get(input).getGeneric(1).getRawClass()) : Object.class; + if (this.kafkaStreamsBindingInformationCatalogue.isUseNativeDecoding( + (KStream) kStreamWrapper)) { + arguments[i] = stream; + } + else { + arguments[i] = this.kafkaStreamsMessageConversionDelegate.deserializeOnInbound( + valueClass, stream); + } + } + + if (arguments[i] == null) { + arguments[i] = stream; + } + Assert.notNull(arguments[i], "Problems encountered while adapting the function argument."); + } + else { + handleKTableGlobalKTableInputs(arguments, i, input, parameterType, targetBean, streamsBuilderFactoryBean, + streamsBuilder, extendedConsumerProperties, keySerde, valueSerde, autoOffsetReset, i == 0); + } + i++; + } + catch (Exception ex) { + throw new IllegalStateException(ex); + } + } + else { + throw new IllegalStateException(StreamListenerErrorMessages.INVALID_DECLARATIVE_METHOD_PARAMETERS); + } + } + return arguments; + } + + @Override + public void setBeanFactory(BeanFactory beanFactory) throws BeansException { + this.beanFactory = beanFactory; + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsJaasConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsJaasConfiguration.java new file mode 100644 index 000000000..8ee3d5257 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsJaasConfiguration.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.io.IOException; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.cloud.stream.binder.kafka.properties.JaasLoginModuleConfiguration; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.security.jaas.KafkaJaasLoginModuleInitializer; + +/** + * Jaas configuration bean for Kafka Streams binder types. + * + * @author Soby Chacko + * @since 3.1.4 + */ +@Configuration +public class KafkaStreamsJaasConfiguration { + + @Bean + @ConditionalOnMissingBean(KafkaJaasLoginModuleInitializer.class) + public KafkaJaasLoginModuleInitializer jaasInitializer( + KafkaBinderConfigurationProperties configurationProperties) + throws IOException { + KafkaJaasLoginModuleInitializer kafkaJaasLoginModuleInitializer = new KafkaJaasLoginModuleInitializer(); + JaasLoginModuleConfiguration jaas = configurationProperties.getJaas(); + if (jaas != null) { + kafkaJaasLoginModuleInitializer.setLoginModule(jaas.getLoginModule()); + + KafkaJaasLoginModuleInitializer.ControlFlag controlFlag = jaas + .getControlFlag(); + + if (controlFlag != null) { + kafkaJaasLoginModuleInitializer.setControlFlag(controlFlag); + } + kafkaJaasLoginModuleInitializer.setOptions(jaas.getOptions()); + } + return kafkaJaasLoginModuleInitializer; + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsMessageConversionDelegate.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsMessageConversionDelegate.java new file mode 100644 index 000000000..78ce848ed --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsMessageConversionDelegate.java @@ -0,0 +1,345 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.processor.Processor; +import org.apache.kafka.streams.processor.ProcessorContext; + +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.converter.CompositeMessageConverter; +import org.springframework.messaging.converter.MessageConverter; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; + +/** + * Delegate for handling all framework level message conversions inbound and outbound on + * {@link KStream}. If native encoding is not enabled, then serialization will be + * performed on outbound messages based on a contentType. Similarly, if native decoding is + * not enabled, deserialization will be performed on inbound messages based on a + * contentType. Based on the contentType, a {@link MessageConverter} will be resolved. + * + * @author Soby Chacko + */ +public class KafkaStreamsMessageConversionDelegate { + + private static final Log LOG = LogFactory + .getLog(KafkaStreamsMessageConversionDelegate.class); + + private static final ThreadLocal> keyValueThreadLocal = new ThreadLocal<>(); + + private final CompositeMessageConverter compositeMessageConverter; + + private final SendToDlqAndContinue sendToDlqAndContinue; + + private final KafkaStreamsBindingInformationCatalogue kstreamBindingInformationCatalogue; + + private final KafkaStreamsBinderConfigurationProperties kstreamBinderConfigurationProperties; + + Exception[] failedWithDeserException = new Exception[1]; + + KafkaStreamsMessageConversionDelegate( + CompositeMessageConverter compositeMessageConverter, + SendToDlqAndContinue sendToDlqAndContinue, + KafkaStreamsBindingInformationCatalogue kstreamBindingInformationCatalogue, + KafkaStreamsBinderConfigurationProperties kstreamBinderConfigurationProperties) { + this.compositeMessageConverter = compositeMessageConverter; + this.sendToDlqAndContinue = sendToDlqAndContinue; + this.kstreamBindingInformationCatalogue = kstreamBindingInformationCatalogue; + this.kstreamBinderConfigurationProperties = kstreamBinderConfigurationProperties; + } + + /** + * Serialize {@link KStream} records on outbound based on contentType. + * @param outboundBindTarget outbound KStream target + * @return serialized KStream + */ + @SuppressWarnings({"rawtypes", "unchecked"}) + public KStream serializeOnOutbound(KStream outboundBindTarget) { + String contentType = this.kstreamBindingInformationCatalogue + .getContentType(outboundBindTarget); + MessageConverter messageConverter = this.compositeMessageConverter; + final PerRecordContentTypeHolder perRecordContentTypeHolder = new PerRecordContentTypeHolder(); + + final KStream kStreamWithEnrichedHeaders = outboundBindTarget + .filter((k, v) -> v != null) + .mapValues((v) -> { + Message message = v instanceof Message ? (Message) v + : MessageBuilder.withPayload(v).build(); + Map headers = new HashMap<>(message.getHeaders()); + if (!StringUtils.isEmpty(contentType)) { + headers.put(MessageHeaders.CONTENT_TYPE, contentType); + } + MessageHeaders messageHeaders = new MessageHeaders(headers); + final Message convertedMessage = messageConverter.toMessage(message.getPayload(), messageHeaders); + perRecordContentTypeHolder.setContentType((String) messageHeaders.get(MessageHeaders.CONTENT_TYPE)); + return convertedMessage.getPayload(); + }); + + kStreamWithEnrichedHeaders.process(() -> new Processor() { + + ProcessorContext context; + + @Override + public void init(ProcessorContext context) { + this.context = context; + } + + @Override + public void process(Object key, Object value) { + if (perRecordContentTypeHolder.contentType != null) { + this.context.headers().remove(MessageHeaders.CONTENT_TYPE); + final Header header; + try { + header = new RecordHeader(MessageHeaders.CONTENT_TYPE, + new ObjectMapper().writeValueAsBytes(perRecordContentTypeHolder.contentType)); + this.context.headers().add(header); + } + catch (Exception e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Could not add content type header"); + } + } + perRecordContentTypeHolder.unsetContentType(); + } + } + + @Override + public void close() { + + } + }); + + return kStreamWithEnrichedHeaders; + } + + /** + * Deserialize incoming {@link KStream} based on content type. + * @param valueClass on KStream value + * @param bindingTarget inbound KStream target + * @return deserialized KStream + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + public KStream deserializeOnInbound(Class valueClass, + KStream bindingTarget) { + MessageConverter messageConverter = this.compositeMessageConverter; + final PerRecordContentTypeHolder perRecordContentTypeHolder = new PerRecordContentTypeHolder(); + + resolvePerRecordContentType(bindingTarget, perRecordContentTypeHolder); + + // Deserialize using a branching strategy + KStream[] branch = bindingTarget.branch( + // First filter where the message is converted and return true if + // everything went well, return false otherwise. + (o, o2) -> { + boolean isValidRecord = false; + + try { + // if the record is a tombstone, ignore and exit from processing + // further. + if (o2 != null) { + if (o2 instanceof Message || o2 instanceof String + || o2 instanceof byte[]) { + Message m1 = null; + if (o2 instanceof Message) { + m1 = perRecordContentTypeHolder.contentType != null + ? MessageBuilder.fromMessage((Message) o2) + .setHeader( + MessageHeaders.CONTENT_TYPE, + perRecordContentTypeHolder.contentType) + .build() + : (Message) o2; + } + else { + m1 = perRecordContentTypeHolder.contentType != null + ? MessageBuilder.withPayload(o2).setHeader( + MessageHeaders.CONTENT_TYPE, + perRecordContentTypeHolder.contentType) + .build() + : MessageBuilder.withPayload(o2).build(); + } + convertAndSetMessage(o, valueClass, messageConverter, m1); + } + else { + keyValueThreadLocal.set(new KeyValue<>(o, o2)); + } + isValidRecord = true; + } + else { + LOG.info( + "Received a tombstone record. This will be skipped from further processing."); + } + } + catch (Exception e) { + LOG.warn( + "Deserialization has failed. This will be skipped from further processing.", + e); + // pass through + failedWithDeserException[0] = e; + } + return isValidRecord; + }, + // second filter that catches any messages for which an exception thrown + // in the first filter above. + (k, v) -> true); + // process errors from the second filter in the branch above. + processErrorFromDeserialization(bindingTarget, branch[1], failedWithDeserException); + + // first branch above is the branch where the messages are converted, let it go + // through further processing. + return branch[0].mapValues((o2) -> { + Object objectValue = keyValueThreadLocal.get().value; + keyValueThreadLocal.remove(); + return objectValue; + }); + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + private void resolvePerRecordContentType(KStream outboundBindTarget, + PerRecordContentTypeHolder perRecordContentTypeHolder) { + outboundBindTarget.process(() -> new Processor() { + + ProcessorContext context; + + @Override + public void init(ProcessorContext context) { + this.context = context; + } + + @Override + public void process(Object key, Object value) { + final Headers headers = this.context.headers(); + final Iterable
contentTypes = headers + .headers(MessageHeaders.CONTENT_TYPE); + if (contentTypes != null && contentTypes.iterator().hasNext()) { + final String contentType = new String( + contentTypes.iterator().next().value()); + // remove leading and trailing quotes + final String cleanContentType = StringUtils.replace(contentType, "\"", + ""); + perRecordContentTypeHolder.setContentType(cleanContentType); + } + } + + @Override + public void close() { + + } + }); + } + + private void convertAndSetMessage(Object o, Class valueClass, + MessageConverter messageConverter, Message msg) { + Object result = valueClass.isAssignableFrom(msg.getPayload().getClass()) + ? msg.getPayload() : messageConverter.fromMessage(msg, valueClass); + + Assert.notNull(result, "Failed to convert message " + msg); + + keyValueThreadLocal.set(new KeyValue<>(o, result)); + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + private void processErrorFromDeserialization(KStream bindingTarget, + KStream branch, Exception[] exception) { + branch.process(() -> new Processor() { + ProcessorContext context; + + @Override + public void init(ProcessorContext context) { + this.context = context; + } + + @Override + public void process(Object o, Object o2) { + // Only continue if the record was not a tombstone. + if (o2 != null) { + if (KafkaStreamsMessageConversionDelegate.this.kstreamBindingInformationCatalogue + .isDlqEnabled(bindingTarget)) { + if (o2 instanceof Message) { + Message message = (Message) o2; + + // We need to convert the key to a byte[] before sending to DLQ. + Serde keySerde = kstreamBindingInformationCatalogue.getKeySerde(bindingTarget); + Serializer keySerializer = keySerde.serializer(); + byte[] keyBytes = keySerializer.serialize(null, o); + + ConsumerRecord consumerRecord = new ConsumerRecord(this.context.topic(), this.context.partition(), this.context.offset(), + keyBytes, message.getPayload()); + + KafkaStreamsMessageConversionDelegate.this.sendToDlqAndContinue + .sendToDlq(consumerRecord, exception[0]); + } + else { + ConsumerRecord consumerRecord = new ConsumerRecord(this.context.topic(), this.context.partition(), this.context.offset(), + o, o2); + KafkaStreamsMessageConversionDelegate.this.sendToDlqAndContinue + .sendToDlq(consumerRecord, exception[0]); + } + } + else if (KafkaStreamsMessageConversionDelegate.this.kstreamBinderConfigurationProperties + .getSerdeError() == KafkaStreamsBinderConfigurationProperties.SerdeError.logAndFail) { + throw new IllegalStateException("Inbound deserialization failed. " + + "Stopping further processing of records."); + } + else if (KafkaStreamsMessageConversionDelegate.this.kstreamBinderConfigurationProperties + .getSerdeError() == KafkaStreamsBinderConfigurationProperties.SerdeError.logAndContinue) { + // quietly passing through. No action needed, this is similar to + // log and continue. + LOG.error( + "Inbound deserialization failed. Skipping this record and continuing."); + } + } + } + + @Override + public void close() { + + } + }); + } + + private static class PerRecordContentTypeHolder { + + String contentType; + + void setContentType(String contentType) { + this.contentType = contentType; + } + + void unsetContentType() { + this.contentType = null; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsRegistry.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsRegistry.java new file mode 100644 index 000000000..ced9ac5ca --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsRegistry.java @@ -0,0 +1,93 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; + +import org.springframework.kafka.config.StreamsBuilderFactoryBean; + +/** + * An internal registry for holding {@link KafkaStreams} objects maintained through + * {@link StreamsBuilderFactoryManager}. + * + * @author Soby Chacko + */ +public class KafkaStreamsRegistry { + + private final Map streamsBuilderFactoryBeanMap = new ConcurrentHashMap<>(); + + private final Set kafkaStreams = ConcurrentHashMap.newKeySet(); + + Set getKafkaStreams() { + Set currentlyRunningKafkaStreams = new HashSet<>(); + for (KafkaStreams ks : this.kafkaStreams) { + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = streamsBuilderFactoryBeanMap.get(ks); + if (streamsBuilderFactoryBean.isRunning()) { + currentlyRunningKafkaStreams.add(ks); + } + } + return currentlyRunningKafkaStreams; + } + + /** + * Register the {@link KafkaStreams} object created in the application. + * @param streamsBuilderFactoryBean {@link StreamsBuilderFactoryBean} + */ + void registerKafkaStreams(StreamsBuilderFactoryBean streamsBuilderFactoryBean) { + final KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams(); + this.kafkaStreams.add(kafkaStreams); + this.streamsBuilderFactoryBeanMap.put(kafkaStreams, streamsBuilderFactoryBean); + } + + void unregisterKafkaStreams(KafkaStreams kafkaStreams) { + this.kafkaStreams.remove(kafkaStreams); + this.streamsBuilderFactoryBeanMap.remove(kafkaStreams); + } + + /** + * + * @param kafkaStreams {@link KafkaStreams} object + * @return Corresponding {@link StreamsBuilderFactoryBean}. + */ + StreamsBuilderFactoryBean streamBuilderFactoryBean(KafkaStreams kafkaStreams) { + return this.streamsBuilderFactoryBeanMap.get(kafkaStreams); + } + + public StreamsBuilderFactoryBean streamsBuilderFactoryBean(String applicationId) { + final Optional first = this.streamsBuilderFactoryBeanMap.values() + .stream() + .filter(streamsBuilderFactoryBean -> streamsBuilderFactoryBean.isRunning() && streamsBuilderFactoryBean + .getStreamsConfiguration().getProperty(StreamsConfig.APPLICATION_ID_CONFIG) + .equals(applicationId)) + .findFirst(); + return first.orElse(null); + } + + public List streamsBuilderFactoryBeans() { + return new ArrayList<>(this.streamsBuilderFactoryBeanMap.values()); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsStreamListenerSetupMethodOrchestrator.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsStreamListenerSetupMethodOrchestrator.java new file mode 100644 index 000000000..6e315faf1 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsStreamListenerSetupMethodOrchestrator.java @@ -0,0 +1,521 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.Topology; +import org.apache.kafka.streams.kstream.GlobalKTable; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.state.StoreBuilder; +import org.apache.kafka.streams.state.Stores; + +import org.springframework.beans.factory.BeanInitializationException; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsStateStore; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsStateStoreProperties; +import org.springframework.cloud.stream.binding.StreamListenerErrorMessages; +import org.springframework.cloud.stream.binding.StreamListenerParameterAdapter; +import org.springframework.cloud.stream.binding.StreamListenerResultAdapter; +import org.springframework.cloud.stream.binding.StreamListenerSetupMethodOrchestrator; +import org.springframework.cloud.stream.config.BindingProperties; +import org.springframework.cloud.stream.config.BindingServiceProperties; +import org.springframework.context.ApplicationContext; +import org.springframework.core.MethodParameter; +import org.springframework.core.ResolvableType; +import org.springframework.core.annotation.AnnotationUtils; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.config.StreamsBuilderFactoryBeanConfigurer; +import org.springframework.kafka.core.CleanupConfig; +import org.springframework.messaging.handler.annotation.SendTo; +import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; +import org.springframework.util.ReflectionUtils; +import org.springframework.util.StringUtils; + +/** + * Kafka Streams specific implementation for {@link StreamListenerSetupMethodOrchestrator} + * that overrides the default mechanisms for invoking StreamListener adapters. + *

+ * The orchestration primarily focus on the following areas: + *

+ * 1. Allow multiple KStream output bindings (KStream branching) by allowing more than one + * output values on {@link SendTo} 2. Allow multiple inbound bindings for multiple KStream + * and or KTable/GlobalKTable types. 3. Each StreamListener method that it orchestrates + * gets its own {@link StreamsBuilderFactoryBean} and {@link StreamsConfig} + * + * @author Soby Chacko + * @author Lei Chen + * @author Gary Russell + */ +class KafkaStreamsStreamListenerSetupMethodOrchestrator extends AbstractKafkaStreamsBinderProcessor + implements StreamListenerSetupMethodOrchestrator { + + private static final Log LOG = LogFactory + .getLog(KafkaStreamsStreamListenerSetupMethodOrchestrator.class); + + private final StreamListenerParameterAdapter streamListenerParameterAdapter; + + private final Collection streamListenerResultAdapters; + + private final BindingServiceProperties bindingServiceProperties; + + private final KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties; + + private final KeyValueSerdeResolver keyValueSerdeResolver; + + private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; + + private final Map> registeredStoresPerMethod = new HashMap<>(); + + private final Map methodStreamsBuilderFactoryBeanMap = new HashMap<>(); + + StreamsBuilderFactoryBeanConfigurer customizer; + + private final ConfigurableEnvironment environment; + + KafkaStreamsStreamListenerSetupMethodOrchestrator( + BindingServiceProperties bindingServiceProperties, + KafkaStreamsExtendedBindingProperties extendedBindingProperties, + KeyValueSerdeResolver keyValueSerdeResolver, + KafkaStreamsBindingInformationCatalogue bindingInformationCatalogue, + StreamListenerParameterAdapter streamListenerParameterAdapter, + Collection listenerResultAdapters, + CleanupConfig cleanupConfig, + StreamsBuilderFactoryBeanConfigurer customizer, + ConfigurableEnvironment environment) { + super(bindingServiceProperties, bindingInformationCatalogue, extendedBindingProperties, keyValueSerdeResolver, cleanupConfig); + this.bindingServiceProperties = bindingServiceProperties; + this.kafkaStreamsExtendedBindingProperties = extendedBindingProperties; + this.keyValueSerdeResolver = keyValueSerdeResolver; + this.kafkaStreamsBindingInformationCatalogue = bindingInformationCatalogue; + this.streamListenerParameterAdapter = streamListenerParameterAdapter; + this.streamListenerResultAdapters = listenerResultAdapters; + this.customizer = customizer; + this.environment = environment; + } + + @Override + public boolean supports(Method method) { + return methodParameterSupports(method) && (methodReturnTypeSuppports(method) + || Void.TYPE.equals(method.getReturnType())); + } + + private boolean methodReturnTypeSuppports(Method method) { + Class returnType = method.getReturnType(); + if (returnType.equals(KStream.class) || (returnType.isArray() + && returnType.getComponentType().equals(KStream.class))) { + return true; + } + return false; + } + + private boolean methodParameterSupports(Method method) { + boolean supports = false; + for (int i = 0; i < method.getParameterCount(); i++) { + MethodParameter methodParameter = MethodParameter.forExecutable(method, i); + Class parameterType = methodParameter.getParameterType(); + if (parameterType.equals(KStream.class) || parameterType.equals(KTable.class) + || parameterType.equals(GlobalKTable.class)) { + supports = true; + } + } + return supports; + } + + @Override + @SuppressWarnings({"rawtypes", "unchecked"}) + public void orchestrateStreamListenerSetupMethod(StreamListener streamListener, + Method method, Object bean) { + String[] methodAnnotatedOutboundNames = getOutboundBindingTargetNames(method); + validateStreamListenerMethod(streamListener, method, + methodAnnotatedOutboundNames); + String methodAnnotatedInboundName = streamListener.value(); + Object[] adaptedInboundArguments = adaptAndRetrieveInboundArguments(method, + methodAnnotatedInboundName, this.applicationContext, + this.streamListenerParameterAdapter); + try { + ReflectionUtils.makeAccessible(method); + if (Void.TYPE.equals(method.getReturnType())) { + method.invoke(bean, adaptedInboundArguments); + } + else { + Object result = method.invoke(bean, adaptedInboundArguments); + + if (methodAnnotatedOutboundNames != null && methodAnnotatedOutboundNames.length > 0) { + if (result.getClass().isArray()) { + Assert.isTrue( + methodAnnotatedOutboundNames.length == ((Object[]) result).length, + "Result does not match with the number of declared outbounds"); + } + else { + Assert.isTrue(methodAnnotatedOutboundNames.length == 1, + "Result does not match with the number of declared outbounds"); + } + } + + if (methodAnnotatedOutboundNames != null && methodAnnotatedOutboundNames.length > 0) { + methodAnnotatedInboundName = populateInboundIfMissing(method, methodAnnotatedInboundName); + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsBindingInformationCatalogue + .getStreamsBuilderFactoryBeanPerBinding().get(methodAnnotatedInboundName); + + if (result.getClass().isArray()) { + Object[] outboundKStreams = (Object[]) result; + int i = 0; + for (Object outboundKStream : outboundKStreams) { + final String methodAnnotatedOutboundName = methodAnnotatedOutboundNames[i++]; + + this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding( + methodAnnotatedOutboundName, streamsBuilderFactoryBean); + + Object targetBean = this.applicationContext + .getBean(methodAnnotatedOutboundName); + kafkaStreamsBindingInformationCatalogue.addOutboundKStreamResolvable(targetBean, ResolvableType.forMethodReturnType(method)); + adaptStreamListenerResult(outboundKStream, targetBean); + } + } + else { + this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding( + methodAnnotatedOutboundNames[0], streamsBuilderFactoryBean); + + Object targetBean = this.applicationContext + .getBean(methodAnnotatedOutboundNames[0]); + kafkaStreamsBindingInformationCatalogue.addOutboundKStreamResolvable(targetBean, ResolvableType.forMethodReturnType(method)); + adaptStreamListenerResult(result, targetBean); + } + } + } + } + catch (Exception ex) { + throw new BeanInitializationException( + "Cannot setup StreamListener for " + method, ex); + } + } + + private String populateInboundIfMissing(Method method, String methodAnnotatedInboundName) { + if (!StringUtils.hasText(methodAnnotatedInboundName)) { + Object[] arguments = new Object[method.getParameterTypes().length]; + if (arguments.length > 0) { + MethodParameter methodParameter = MethodParameter.forExecutable(method, 0); + if (methodParameter.hasParameterAnnotation(Input.class)) { + Input methodAnnotation = methodParameter + .getParameterAnnotation(Input.class); + methodAnnotatedInboundName = methodAnnotation.value(); + } + } + } + return methodAnnotatedInboundName; + } + + @SuppressWarnings("unchecked") + private void adaptStreamListenerResult(Object outboundKStream, Object targetBean) { + for (StreamListenerResultAdapter streamListenerResultAdapter : this.streamListenerResultAdapters) { + if (streamListenerResultAdapter.supports( + outboundKStream.getClass(), targetBean.getClass())) { + streamListenerResultAdapter.adapt(outboundKStream, + targetBean); + break; + } + } + } + + @Override + @SuppressWarnings({"unchecked"}) + public Object[] adaptAndRetrieveInboundArguments(Method method, String inboundName, + ApplicationContext applicationContext, + StreamListenerParameterAdapter... adapters) { + Object[] arguments = new Object[method.getParameterTypes().length]; + for (int parameterIndex = 0; parameterIndex < arguments.length; parameterIndex++) { + MethodParameter methodParameter = MethodParameter.forExecutable(method, + parameterIndex); + Class parameterType = methodParameter.getParameterType(); + Object targetReferenceValue = null; + if (methodParameter.hasParameterAnnotation(Input.class)) { + targetReferenceValue = AnnotationUtils + .getValue(methodParameter.getParameterAnnotation(Input.class)); + Input methodAnnotation = methodParameter + .getParameterAnnotation(Input.class); + inboundName = methodAnnotation.value(); + } + else if (arguments.length == 1 && StringUtils.hasText(inboundName)) { + targetReferenceValue = inboundName; + } + if (targetReferenceValue != null) { + Assert.isInstanceOf(String.class, targetReferenceValue, + "Annotation value must be a String"); + Object targetBean = applicationContext + .getBean((String) targetReferenceValue); + BindingProperties bindingProperties = this.bindingServiceProperties + .getBindingProperties(inboundName); + // Retrieve the StreamsConfig created for this method if available. + // Otherwise, create the StreamsBuilderFactory and get the underlying + // config. + if (!this.methodStreamsBuilderFactoryBeanMap.containsKey(method)) { + StreamsBuilderFactoryBean streamsBuilderFactoryBean = buildStreamsBuilderAndRetrieveConfig(method.getDeclaringClass().getSimpleName() + "-" + method.getName(), + applicationContext, + inboundName, null, customizer, this.environment, bindingProperties); + this.methodStreamsBuilderFactoryBeanMap.put(method, streamsBuilderFactoryBean); + } + try { + StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.methodStreamsBuilderFactoryBeanMap + .get(method); + StreamsBuilder streamsBuilder = streamsBuilderFactoryBean.getObject(); + final String applicationId = streamsBuilderFactoryBean.getStreamsConfiguration().getProperty(StreamsConfig.APPLICATION_ID_CONFIG); + KafkaStreamsConsumerProperties extendedConsumerProperties = this.kafkaStreamsExtendedBindingProperties + .getExtendedConsumerProperties(inboundName); + extendedConsumerProperties.setApplicationId(applicationId); + // get state store spec + KafkaStreamsStateStoreProperties spec = buildStateStoreSpec(method); + + Serde keySerde = this.keyValueSerdeResolver + .getInboundKeySerde(extendedConsumerProperties, ResolvableType.forMethodParameter(methodParameter)); + LOG.info("Key Serde used for " + targetReferenceValue + ": " + keySerde.getClass().getName()); + + Serde valueSerde = bindingServiceProperties.getConsumerProperties(inboundName).isUseNativeDecoding() ? + getValueSerde(inboundName, extendedConsumerProperties, ResolvableType.forMethodParameter(methodParameter)) : Serdes.ByteArray(); + LOG.info("Value Serde used for " + targetReferenceValue + ": " + valueSerde.getClass().getName()); + + Topology.AutoOffsetReset autoOffsetReset = getAutoOffsetReset(inboundName, extendedConsumerProperties); + + if (parameterType.isAssignableFrom(KStream.class)) { + KStream stream = getkStream(inboundName, spec, + bindingProperties, extendedConsumerProperties, streamsBuilder, keySerde, valueSerde, + autoOffsetReset, parameterIndex == 0); + KStreamBoundElementFactory.KStreamWrapper kStreamWrapper = (KStreamBoundElementFactory.KStreamWrapper) targetBean; + // wrap the proxy created during the initial target type binding + // with real object (KStream) + kStreamWrapper.wrap((KStream) stream); + this.kafkaStreamsBindingInformationCatalogue.addKeySerde(stream, keySerde); + BindingProperties bindingProperties1 = this.kafkaStreamsBindingInformationCatalogue.getBindingProperties().get(kStreamWrapper); + this.kafkaStreamsBindingInformationCatalogue.registerBindingProperties(stream, bindingProperties1); + + this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding(inboundName, streamsBuilderFactoryBean); + this.kafkaStreamsBindingInformationCatalogue.addConsumerPropertiesPerSbfb(streamsBuilderFactoryBean, + bindingServiceProperties.getConsumerProperties(inboundName)); + + for (StreamListenerParameterAdapter streamListenerParameterAdapter : adapters) { + if (streamListenerParameterAdapter.supports(stream.getClass(), + methodParameter)) { + arguments[parameterIndex] = streamListenerParameterAdapter + .adapt(stream, methodParameter); + break; + } + } + if (arguments[parameterIndex] == null + && parameterType.isAssignableFrom(stream.getClass())) { + arguments[parameterIndex] = stream; + } + Assert.notNull(arguments[parameterIndex], + "Cannot convert argument " + parameterIndex + " of " + + method + "from " + stream.getClass() + " to " + + parameterType); + } + else { + handleKTableGlobalKTableInputs(arguments, parameterIndex, inboundName, parameterType, targetBean, streamsBuilderFactoryBean, + streamsBuilder, extendedConsumerProperties, keySerde, valueSerde, autoOffsetReset, parameterIndex == 0); + } + } + catch (Exception ex) { + throw new IllegalStateException(ex); + } + } + else { + throw new IllegalStateException( + StreamListenerErrorMessages.INVALID_DECLARATIVE_METHOD_PARAMETERS); + } + } + return arguments; + } + + private StoreBuilder buildStateStore(KafkaStreamsStateStoreProperties spec) { + try { + + Serde keySerde = this.keyValueSerdeResolver + .getStateStoreKeySerde(spec.getKeySerdeString()); + Serde valueSerde = this.keyValueSerdeResolver + .getStateStoreValueSerde(spec.getValueSerdeString()); + StoreBuilder builder; + switch (spec.getType()) { + case KEYVALUE: + builder = Stores.keyValueStoreBuilder( + Stores.persistentKeyValueStore(spec.getName()), keySerde, + valueSerde); + break; + case WINDOW: + builder = Stores + .windowStoreBuilder( + Stores.persistentWindowStore(spec.getName(), + Duration.ofMillis(spec.getRetention()), Duration.ofMillis(3), false), + keySerde, valueSerde); + break; + case SESSION: + builder = Stores.sessionStoreBuilder(Stores.persistentSessionStore( + spec.getName(), Duration.ofMillis(spec.getRetention())), keySerde, valueSerde); + break; + default: + throw new UnsupportedOperationException( + "state store type (" + spec.getType() + ") is not supported!"); + } + if (spec.isCacheEnabled()) { + builder = builder.withCachingEnabled(); + } + if (spec.isLoggingDisabled()) { + builder = builder.withLoggingDisabled(); + } + return builder; + } + catch (Exception ex) { + LOG.error("failed to build state store exception : " + ex); + throw ex; + } + } + + private KStream getkStream(String inboundName, + KafkaStreamsStateStoreProperties storeSpec, + BindingProperties bindingProperties, + KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, StreamsBuilder streamsBuilder, + Serde keySerde, Serde valueSerde, + Topology.AutoOffsetReset autoOffsetReset, boolean firstBuild) { + if (storeSpec != null) { + StoreBuilder storeBuilder = buildStateStore(storeSpec); + streamsBuilder.addStateStore(storeBuilder); + if (LOG.isInfoEnabled()) { + LOG.info("state store " + storeBuilder.name() + " added to topology"); + } + } + return getKStream(inboundName, bindingProperties, kafkaStreamsConsumerProperties, streamsBuilder, + keySerde, valueSerde, autoOffsetReset, firstBuild); + } + + private void validateStreamListenerMethod(StreamListener streamListener, + Method method, String[] methodAnnotatedOutboundNames) { + String methodAnnotatedInboundName = streamListener.value(); + if (methodAnnotatedOutboundNames != null) { + for (String s : methodAnnotatedOutboundNames) { + if (StringUtils.hasText(s)) { + Assert.isTrue(isDeclarativeOutput(method, s), + "Method must be declarative"); + } + } + } + if (StringUtils.hasText(methodAnnotatedInboundName)) { + int methodArgumentsLength = method.getParameterTypes().length; + + for (int parameterIndex = 0; parameterIndex < methodArgumentsLength; parameterIndex++) { + MethodParameter methodParameter = MethodParameter.forExecutable(method, + parameterIndex); + Assert.isTrue( + isDeclarativeInput(methodAnnotatedInboundName, methodParameter), + "Method must be declarative"); + } + } + } + + @SuppressWarnings("unchecked") + private boolean isDeclarativeOutput(Method m, String targetBeanName) { + boolean declarative; + Class returnType = m.getReturnType(); + if (returnType.isArray()) { + Class targetBeanClass = this.applicationContext.getType(targetBeanName); + declarative = this.streamListenerResultAdapters.stream() + .anyMatch((slpa) -> slpa.supports(returnType.getComponentType(), + targetBeanClass)); + return declarative; + } + Class targetBeanClass = this.applicationContext.getType(targetBeanName); + declarative = this.streamListenerResultAdapters.stream() + .anyMatch((slpa) -> slpa.supports(returnType, targetBeanClass)); + return declarative; + } + + @SuppressWarnings("unchecked") + private boolean isDeclarativeInput(String targetBeanName, + MethodParameter methodParameter) { + if (!methodParameter.getParameterType().isAssignableFrom(Object.class) + && this.applicationContext.containsBean(targetBeanName)) { + Class targetBeanClass = this.applicationContext.getType(targetBeanName); + if (targetBeanClass != null) { + boolean supports = KafkaStreamsBinderUtils.supportsKStream(methodParameter, targetBeanClass); + if (!supports) { + supports = KTable.class.isAssignableFrom(targetBeanClass) + && KTable.class.isAssignableFrom(methodParameter.getParameterType()); + if (!supports) { + supports = GlobalKTable.class.isAssignableFrom(targetBeanClass) + && GlobalKTable.class.isAssignableFrom(methodParameter.getParameterType()); + } + } + return supports; + } + } + return false; + } + + private static String[] getOutboundBindingTargetNames(Method method) { + SendTo sendTo = AnnotationUtils.findAnnotation(method, SendTo.class); + if (sendTo != null) { + Assert.isTrue(!ObjectUtils.isEmpty(sendTo.value()), + StreamListenerErrorMessages.ATLEAST_ONE_OUTPUT); + Assert.isTrue(sendTo.value().length >= 1, + "At least one outbound destination need to be provided."); + return sendTo.value(); + } + return null; + } + + @SuppressWarnings({"unchecked"}) + private KafkaStreamsStateStoreProperties buildStateStoreSpec(Method method) { + if (!this.registeredStoresPerMethod.containsKey(method)) { + KafkaStreamsStateStore spec = AnnotationUtils.findAnnotation(method, + KafkaStreamsStateStore.class); + if (spec != null) { + Assert.isTrue(!ObjectUtils.isEmpty(spec.name()), "name cannot be empty"); + Assert.isTrue(spec.name().length() >= 1, "name cannot be empty."); + this.registeredStoresPerMethod.put(method, new ArrayList<>()); + this.registeredStoresPerMethod.get(method).add(spec.name()); + KafkaStreamsStateStoreProperties props = new KafkaStreamsStateStoreProperties(); + props.setName(spec.name()); + props.setType(spec.type()); + props.setLength(spec.lengthMs()); + props.setKeySerdeString(spec.keySerde()); + props.setRetention(spec.retentionMs()); + props.setValueSerdeString(spec.valueSerde()); + props.setCacheEnabled(spec.cache()); + props.setLoggingDisabled(!spec.logging()); + return props; + } + } + return null; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KeyValueSerdeResolver.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KeyValueSerdeResolver.java new file mode 100644 index 000000000..b23d51393 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KeyValueSerdeResolver.java @@ -0,0 +1,445 @@ +/* + * Copyright 2018-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.streams.kstream.GlobalKTable; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.annotation.AnnotatedBeanDefinition; +import org.springframework.cloud.stream.binder.ConsumerProperties; +import org.springframework.cloud.stream.binder.ProducerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsProducerProperties; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.core.ResolvableType; +import org.springframework.kafka.support.serializer.JsonSerde; +import org.springframework.util.ClassUtils; +import org.springframework.util.StringUtils; + +/** + * Resolver for key and value Serde. + * + * On the inbound, if native decoding is enabled, then any deserialization on the value is + * handled by Kafka. First, we look for any key/value Serde set on the binding itself, if + * that is not available then look at the common Serde set at the global level. If that + * fails, it falls back to byte[]. If native decoding is disabled, then the binder will do + * the deserialization on value and ignore any Serde set for value and rely on the + * contentType provided. Keys are always deserialized at the broker. + * + * + * Same rules apply on the outbound. If native encoding is enabled, then value + * serialization is done at the broker using any binder level Serde for value, if not + * using common Serde, if not, then byte[]. If native encoding is disabled, then the + * binder will do serialization using a contentType. Keys are always serialized by the + * broker. + * + * For state store, use serdes class specified in + * {@link org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsStateStore} + * to create Serde accordingly. + * + * @author Soby Chacko + * @author Lei Chen + * @author Eduard Domínguez + */ +public class KeyValueSerdeResolver implements ApplicationContextAware { + + private static final Log LOG = LogFactory.getLog(KeyValueSerdeResolver.class); + + private final Map streamConfigGlobalProperties; + + private final KafkaStreamsBinderConfigurationProperties binderConfigurationProperties; + + private ConfigurableApplicationContext context; + + KeyValueSerdeResolver(Map streamConfigGlobalProperties, + KafkaStreamsBinderConfigurationProperties binderConfigurationProperties) { + this.streamConfigGlobalProperties = streamConfigGlobalProperties; + this.binderConfigurationProperties = binderConfigurationProperties; + } + + /** + * Provide the {@link Serde} for inbound key. + * @param extendedConsumerProperties binding level extended + * {@link KafkaStreamsConsumerProperties} + * @return configurd {@link Serde} for the inbound key. + */ + public Serde getInboundKeySerde( + KafkaStreamsConsumerProperties extendedConsumerProperties) { + String keySerdeString = extendedConsumerProperties.getKeySerde(); + + return getKeySerde(keySerdeString, extendedConsumerProperties.getConfiguration()); + } + + public Serde getInboundKeySerde( + KafkaStreamsConsumerProperties extendedConsumerProperties, ResolvableType resolvableType) { + String keySerdeString = extendedConsumerProperties.getKeySerde(); + + return getKeySerde(keySerdeString, resolvableType, extendedConsumerProperties.getConfiguration()); + } + + /** + * Provide the {@link Serde} for inbound value. + * @param consumerProperties {@link ConsumerProperties} on binding + * @param extendedConsumerProperties binding level extended + * {@link KafkaStreamsConsumerProperties} + * @return configurd {@link Serde} for the inbound value. + */ + public Serde getInboundValueSerde(ConsumerProperties consumerProperties, + KafkaStreamsConsumerProperties extendedConsumerProperties) { + Serde valueSerde; + + String valueSerdeString = extendedConsumerProperties.getValueSerde(); + try { + if (consumerProperties != null && consumerProperties.isUseNativeDecoding()) { + valueSerde = getValueSerde(valueSerdeString, extendedConsumerProperties.getConfiguration()); + } + else { + valueSerde = Serdes.ByteArray(); + } + } + catch (ClassNotFoundException ex) { + throw new IllegalStateException("Serde class not found: ", ex); + } + return valueSerde; + } + + public Serde getInboundValueSerde(ConsumerProperties consumerProperties, + KafkaStreamsConsumerProperties extendedConsumerProperties, + ResolvableType resolvableType) { + Serde valueSerde; + + String valueSerdeString = extendedConsumerProperties.getValueSerde(); + try { + if (consumerProperties != null && consumerProperties.isUseNativeDecoding()) { + valueSerde = getValueSerde(valueSerdeString, resolvableType, extendedConsumerProperties.getConfiguration()); + } + else { + valueSerde = Serdes.ByteArray(); + } + } + catch (ClassNotFoundException ex) { + throw new IllegalStateException("Serde class not found: ", ex); + } + return valueSerde; + } + + /** + * Provide the {@link Serde} for outbound key. + * @param properties binding level extended {@link KafkaStreamsProducerProperties} + * @return configurd {@link Serde} for the outbound key. + */ + public Serde getOuboundKeySerde(KafkaStreamsProducerProperties properties) { + return getKeySerde(properties.getKeySerde(), properties.getConfiguration()); + } + + public Serde getOuboundKeySerde(KafkaStreamsProducerProperties properties, ResolvableType resolvableType) { + return getKeySerde(properties.getKeySerde(), resolvableType, properties.getConfiguration()); + } + + + /** + * Provide the {@link Serde} for outbound value. + * @param producerProperties {@link ProducerProperties} on binding + * @param kafkaStreamsProducerProperties binding level extended + * {@link KafkaStreamsProducerProperties} + * @return configurd {@link Serde} for the outbound value. + */ + public Serde getOutboundValueSerde(ProducerProperties producerProperties, + KafkaStreamsProducerProperties kafkaStreamsProducerProperties) { + Serde valueSerde; + try { + if (producerProperties.isUseNativeEncoding()) { + valueSerde = getValueSerde( + kafkaStreamsProducerProperties.getValueSerde(), kafkaStreamsProducerProperties.getConfiguration()); + } + else { + valueSerde = Serdes.ByteArray(); + } + } + catch (ClassNotFoundException ex) { + throw new IllegalStateException("Serde class not found: ", ex); + } + return valueSerde; + } + + public Serde getOutboundValueSerde(ProducerProperties producerProperties, + KafkaStreamsProducerProperties kafkaStreamsProducerProperties, ResolvableType resolvableType) { + Serde valueSerde; + try { + if (producerProperties.isUseNativeEncoding()) { + valueSerde = getValueSerde( + kafkaStreamsProducerProperties.getValueSerde(), resolvableType, kafkaStreamsProducerProperties.getConfiguration()); + } + else { + valueSerde = Serdes.ByteArray(); + } + } + catch (ClassNotFoundException ex) { + throw new IllegalStateException("Serde class not found: ", ex); + } + return valueSerde; + } + + /** + * Provide the {@link Serde} for state store. + * @param keySerdeString serde class used for key + * @return {@link Serde} for the state store key. + */ + public Serde getStateStoreKeySerde(String keySerdeString) { + return getKeySerde(keySerdeString, (Map) null); + } + + /** + * Provide the {@link Serde} for state store value. + * @param valueSerdeString serde class used for value + * @return {@link Serde} for the state store value. + */ + public Serde getStateStoreValueSerde(String valueSerdeString) { + try { + return getValueSerde(valueSerdeString, (Map) null); + } + catch (ClassNotFoundException ex) { + throw new IllegalStateException("Serde class not found: ", ex); + } + } + + private Serde getKeySerde(String keySerdeString, Map extendedConfiguration) { + Serde keySerde; + try { + if (StringUtils.hasText(keySerdeString)) { + keySerde = Utils.newInstance(keySerdeString, Serde.class); + } + else { + keySerde = getFallbackSerde("default.key.serde"); + } + keySerde.configure(combineStreamConfigProperties(extendedConfiguration), true); + } + catch (ClassNotFoundException ex) { + throw new IllegalStateException("Serde class not found: ", ex); + } + return keySerde; + } + + private Serde getKeySerde(String keySerdeString, ResolvableType resolvableType, Map extendedConfiguration) { + Serde keySerde = null; + try { + if (StringUtils.hasText(keySerdeString)) { + keySerde = Utils.newInstance(keySerdeString, Serde.class); + } + else { + if (resolvableType != null && + (isResolvalbeKafkaStreamsType(resolvableType) || isResolvableKStreamArrayType(resolvableType))) { + ResolvableType generic = resolvableType.isArray() ? resolvableType.getComponentType().getGeneric(0) : resolvableType.getGeneric(0); + Serde fallbackSerde = getFallbackSerde("default.key.serde"); + keySerde = getSerde(generic, fallbackSerde); + } + if (keySerde == null) { + keySerde = Serdes.ByteArray(); + } + } + keySerde.configure(combineStreamConfigProperties(extendedConfiguration), true); + } + catch (ClassNotFoundException ex) { + throw new IllegalStateException("Serde class not found: ", ex); + } + return keySerde; + } + + private boolean isResolvableKStreamArrayType(ResolvableType resolvableType) { + return resolvableType.isArray() && + KStream.class.isAssignableFrom(resolvableType.getComponentType().getRawClass()); + } + + private boolean isResolvalbeKafkaStreamsType(ResolvableType resolvableType) { + return resolvableType.getRawClass() != null && (KStream.class.isAssignableFrom(resolvableType.getRawClass()) || KTable.class.isAssignableFrom(resolvableType.getRawClass()) || + GlobalKTable.class.isAssignableFrom(resolvableType.getRawClass())); + } + + private Serde getSerde(ResolvableType generic, Serde fallbackSerde) { + Serde serde = null; + + Map beansOfType = context.getBeansOfType(Serde.class); + Serde[] serdeBeans = new Serde[1]; + + final Class genericRawClazz = generic.getRawClass(); + beansOfType.forEach((k, v) -> { + final Class classObj = ClassUtils.resolveClassName(((AnnotatedBeanDefinition) + context.getBeanFactory().getBeanDefinition(k)) + .getMetadata().getClassName(), + ClassUtils.getDefaultClassLoader()); + try { + Method[] methods = classObj.getMethods(); + Optional serdeBeanMethod = Arrays.stream(methods).filter(m -> m.getName().equals(k)).findFirst(); + if (serdeBeanMethod.isPresent()) { + Method method = serdeBeanMethod.get(); + ResolvableType resolvableType = ResolvableType.forMethodReturnType(method, classObj); + ResolvableType serdeBeanGeneric = resolvableType.getGeneric(0); + Class serdeGenericRawClazz = serdeBeanGeneric.getRawClass(); + if (serdeGenericRawClazz != null && genericRawClazz != null) { + if (serdeGenericRawClazz.isAssignableFrom(genericRawClazz)) { + serdeBeans[0] = v; + } + } + } + } + catch (Exception e) { + // Pass through... + } + + }); + + if (serdeBeans[0] != null) { + return serdeBeans[0]; + } + + if (genericRawClazz != null) { + if (Integer.class.isAssignableFrom(genericRawClazz)) { + serde = Serdes.Integer(); + } + else if (Long.class.isAssignableFrom(genericRawClazz)) { + serde = Serdes.Long(); + } + else if (Short.class.isAssignableFrom(genericRawClazz)) { + serde = Serdes.Short(); + } + else if (Double.class.isAssignableFrom(genericRawClazz)) { + serde = Serdes.Double(); + } + else if (Float.class.isAssignableFrom(genericRawClazz)) { + serde = Serdes.Float(); + } + else if (byte[].class.isAssignableFrom(genericRawClazz)) { + serde = Serdes.ByteArray(); + } + else if (String.class.isAssignableFrom(genericRawClazz)) { + serde = Serdes.String(); + } + else if (UUID.class.isAssignableFrom(genericRawClazz)) { + serde = Serdes.UUID(); + } + else if (!isSerdeFromStandardDefaults(fallbackSerde)) { + //User purposely set a default serde that is not one of the above + serde = fallbackSerde; + } + else { + // If the type is Object, then skip assigning the JsonSerde and let the fallback mechanism takes precedence. + if (!genericRawClazz.isAssignableFrom((Object.class))) { + serde = new JsonSerde(genericRawClazz); + } + } + } + return serde; + } + + private boolean isSerdeFromStandardDefaults(Serde serde) { + if (serde != null) { + if (Number.class.isAssignableFrom(serde.getClass())) { + return true; + } + else if (Serdes.ByteArray().getClass().isAssignableFrom(serde.getClass())) { + return true; + } + else if (Serdes.String().getClass().isAssignableFrom(serde.getClass())) { + return true; + } + else if (Serdes.UUID().getClass().isAssignableFrom(serde.getClass())) { + return true; + } + } + return false; + } + + + private Serde getValueSerde(String valueSerdeString, Map extendedConfiguration) + throws ClassNotFoundException { + Serde valueSerde; + if (StringUtils.hasText(valueSerdeString)) { + valueSerde = Utils.newInstance(valueSerdeString, Serde.class); + } + else { + valueSerde = getFallbackSerde("default.value.serde"); + } + valueSerde.configure(combineStreamConfigProperties(extendedConfiguration), false); + return valueSerde; + } + + private Serde getFallbackSerde(String s) throws ClassNotFoundException { + return this.binderConfigurationProperties.getConfiguration() + .containsKey(s) + ? Utils.newInstance(this.binderConfigurationProperties + .getConfiguration().get(s), + Serde.class) + : Serdes.ByteArray(); + } + + @SuppressWarnings("unchecked") + private Serde getValueSerde(String valueSerdeString, ResolvableType resolvableType, Map extendedConfiguration) + throws ClassNotFoundException { + Serde valueSerde = null; + if (StringUtils.hasText(valueSerdeString)) { + valueSerde = Utils.newInstance(valueSerdeString, Serde.class); + } + else { + + if (resolvableType != null && ((isResolvalbeKafkaStreamsType(resolvableType)) || + (isResolvableKStreamArrayType(resolvableType)))) { + Serde fallbackSerde = getFallbackSerde("default.value.serde"); + ResolvableType generic = resolvableType.isArray() ? resolvableType.getComponentType().getGeneric(1) : resolvableType.getGeneric(1); + valueSerde = getSerde(generic, fallbackSerde); + } + if (valueSerde == null) { + + valueSerde = Serdes.ByteArray(); + } + } + valueSerde.configure(combineStreamConfigProperties(extendedConfiguration), false); + return valueSerde; + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + context = (ConfigurableApplicationContext) applicationContext; + } + + private Map combineStreamConfigProperties(Map extendedConfiguration) { + if (extendedConfiguration != null && !extendedConfiguration.isEmpty()) { + Map streamConfiguration = new HashMap(this.streamConfigGlobalProperties); + streamConfiguration.putAll(extendedConfiguration); + return streamConfiguration; + } + else { + return this.streamConfigGlobalProperties; + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/MultiBinderPropertiesConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/MultiBinderPropertiesConfiguration.java new file mode 100644 index 000000000..c0a07f7a9 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/MultiBinderPropertiesConfiguration.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author Soby Chacko + * @since 3.0.2 + */ +@Configuration +public class MultiBinderPropertiesConfiguration { + + @Bean + @ConfigurationProperties(prefix = "spring.cloud.stream.kafka.streams.binder") + @ConditionalOnBean(name = "outerContext") + public KafkaBinderConfigurationProperties binderConfigurationProperties(KafkaProperties kafkaProperties) { + return new KafkaStreamsBinderConfigurationProperties(kafkaProperties); + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/SendToDlqAndContinue.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/SendToDlqAndContinue.java new file mode 100644 index 000000000..569ef8c4f --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/SendToDlqAndContinue.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.kafka.clients.consumer.ConsumerRecord; + +import org.springframework.kafka.listener.ConsumerRecordRecoverer; +import org.springframework.kafka.listener.DeadLetterPublishingRecoverer; + +/** + * Custom implementation for {@link ConsumerRecordRecoverer} that keeps a collection of + * recoverer objects per input topics. These topics might be per input binding or multiplexed + * topics in a single binding. + * + * @author Soby Chacko + * @since 2.0.0 + */ +public class SendToDlqAndContinue implements ConsumerRecordRecoverer { + + /** + * DLQ dispatcher per topic in the application context. The key here is not the actual + * DLQ topic but the incoming topic that caused the error. + */ + private Map dlqDispatchers = new HashMap<>(); + + /** + * For a given topic, send the key/value record to DLQ topic. + * + * @param consumerRecord consumer record + * @param exception exception + */ + public void sendToDlq(ConsumerRecord consumerRecord, Exception exception) { + DeadLetterPublishingRecoverer kafkaStreamsDlqDispatch = this.dlqDispatchers.get(consumerRecord.topic()); + kafkaStreamsDlqDispatch.accept(consumerRecord, exception); + } + + void addKStreamDlqDispatch(String topic, + DeadLetterPublishingRecoverer kafkaStreamsDlqDispatch) { + this.dlqDispatchers.put(topic, kafkaStreamsDlqDispatch); + } + + @Override + public void accept(ConsumerRecord consumerRecord, Exception e) { + this.dlqDispatchers.get(consumerRecord.topic()).accept(consumerRecord, e); + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/SkipAndContinueExceptionHandler.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/SkipAndContinueExceptionHandler.java new file mode 100644 index 000000000..6fff5a25e --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/SkipAndContinueExceptionHandler.java @@ -0,0 +1,46 @@ +/* + * Copyright 2021-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.Map; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.streams.errors.DeserializationExceptionHandler; +import org.apache.kafka.streams.processor.ProcessorContext; + +/** + * + * {@link DeserializationExceptionHandler} that allows to silently skip + * deserialization exceptions and continue processing. + * + * @author Soby Chakco + * @since 3.1.2 + */ +public class SkipAndContinueExceptionHandler implements DeserializationExceptionHandler { + + @Override + public DeserializationExceptionHandler.DeserializationHandlerResponse handle(final ProcessorContext context, + final ConsumerRecord record, + final Exception exception) { + return DeserializationExceptionHandler.DeserializationHandlerResponse.CONTINUE; + } + + @Override + public void configure(final Map configs) { + // ignore + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/StreamsBuilderFactoryManager.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/StreamsBuilderFactoryManager.java new file mode 100644 index 000000000..b8c0277df --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/StreamsBuilderFactoryManager.java @@ -0,0 +1,158 @@ +/* + * Copyright 2018-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler; + +import org.springframework.beans.factory.DisposableBean; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.cloud.stream.binder.ConsumerProperties; +import org.springframework.context.SmartLifecycle; +import org.springframework.kafka.KafkaException; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.streams.KafkaStreamsMicrometerListener; + +/** + * Iterate through all {@link StreamsBuilderFactoryBean} in the application context and + * start them. As each one completes starting, register the associated KafkaStreams object + * into {@link InteractiveQueryService}. + * + * This {@link SmartLifecycle} class ensures that the bean created from it is started very + * late through the bootstrap process by setting the phase value closer to + * Integer.MAX_VALUE. This is to guarantee that the {@link StreamsBuilderFactoryBean} on a + * {@link org.springframework.cloud.stream.annotation.StreamListener} method with multiple + * bindings is only started after all the binding phases have completed successfully. + * + * @author Soby Chacko + */ +public class StreamsBuilderFactoryManager implements SmartLifecycle { + + private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; + + private final KafkaStreamsRegistry kafkaStreamsRegistry; + + private final KafkaStreamsBinderMetrics kafkaStreamsBinderMetrics; + + private final KafkaStreamsMicrometerListener listener; + + private volatile boolean running; + + private final KafkaProperties kafkaProperties; + + StreamsBuilderFactoryManager(KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, + KafkaStreamsRegistry kafkaStreamsRegistry, + KafkaStreamsBinderMetrics kafkaStreamsBinderMetrics, + KafkaStreamsMicrometerListener listener, + KafkaProperties kafkaProperties) { + this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue; + this.kafkaStreamsRegistry = kafkaStreamsRegistry; + this.kafkaStreamsBinderMetrics = kafkaStreamsBinderMetrics; + this.listener = listener; + this.kafkaProperties = kafkaProperties; + } + + @Override + public boolean isAutoStartup() { + return this.kafkaProperties == null || this.kafkaProperties.getStreams().isAutoStartup(); + } + + @Override + public void stop(Runnable callback) { + stop(); + if (callback != null) { + callback.run(); + } + } + + @Override + public synchronized void start() { + if (!this.running) { + try { + Set streamsBuilderFactoryBeans = this.kafkaStreamsBindingInformationCatalogue + .getStreamsBuilderFactoryBeans(); + for (StreamsBuilderFactoryBean streamsBuilderFactoryBean : streamsBuilderFactoryBeans) { + if (this.listener != null) { + streamsBuilderFactoryBean.addListener(this.listener); + } + // By default, we shut down the client if there is an uncaught exception in the application. + // Users can override this by customizing SBFB. See this issue for more details: + // https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1110 + if (streamsBuilderFactoryBean.getStreamsUncaughtExceptionHandler() == null) { + streamsBuilderFactoryBean.setStreamsUncaughtExceptionHandler(exception -> + StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse.SHUTDOWN_CLIENT); + } + // Starting the stream. + final Map> bindingServicePropertiesPerSbfb = + this.kafkaStreamsBindingInformationCatalogue.getConsumerPropertiesPerSbfb(); + final List consumerProperties = bindingServicePropertiesPerSbfb.get(streamsBuilderFactoryBean); + final boolean autoStartupDisabledOnAtLeastOneConsumerBinding = consumerProperties.stream().anyMatch(consumerProperties1 -> !consumerProperties1.isAutoStartup()); + if (!autoStartupDisabledOnAtLeastOneConsumerBinding) { + streamsBuilderFactoryBean.start(); + this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean); + } + } + if (this.kafkaStreamsBinderMetrics != null) { + this.kafkaStreamsBinderMetrics.addMetrics(streamsBuilderFactoryBeans); + } + this.running = true; + } + catch (Exception ex) { + throw new KafkaException("Could not start stream: ", ex); + } + } + } + + @Override + public synchronized void stop() { + if (this.running) { + try { + Set streamsBuilderFactoryBeans = this.kafkaStreamsBindingInformationCatalogue + .getStreamsBuilderFactoryBeans(); + int n = 0; + for (StreamsBuilderFactoryBean streamsBuilderFactoryBean : streamsBuilderFactoryBeans) { + streamsBuilderFactoryBean.removeListener(this.listener); + streamsBuilderFactoryBean.stop(); + } + for (ProducerFactory dlqProducerFactory : this.kafkaStreamsBindingInformationCatalogue.getDlqProducerFactories()) { + ((DisposableBean) dlqProducerFactory).destroy(); + } + } + catch (Exception ex) { + throw new IllegalStateException(ex); + } + finally { + this.running = false; + } + } + } + + @Override + public synchronized boolean isRunning() { + return this.running; + } + + @Override + public int getPhase() { + return Integer.MAX_VALUE - 100; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsProcessor.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsProcessor.java new file mode 100644 index 000000000..36df1f43f --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsProcessor.java @@ -0,0 +1,90 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.annotations; + +import org.apache.kafka.streams.kstream.KStream; + +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.Output; + +/** + * Bindable interface for {@link KStream} input and output. + * + * This interface can be used as a bindable interface with + * {@link org.springframework.cloud.stream.annotation.EnableBinding} when both input and + * output types are single KStream. In other scenarios where multiple types are required, + * other similar bindable interfaces can be created and used. For example, there are cases + * in which multiple KStreams are required on the outbound in the case of KStream + * branching or multiple input types are required either in the form of multiple KStreams + * and a combination of KStreams and KTables. In those cases, new bindable interfaces + * compatible with the requirements must be created. Here are some examples. + * + *

+ *     interface KStreamBranchProcessor {
+ *         @Input("input")
+ *         KStream<?, ?> input();
+ *
+ *         @Output("output-1")
+ *         KStream<?, ?> output1();
+ *
+ *         @Output("output-2")
+ *         KStream<?, ?> output2();
+ *
+ *         @Output("output-3")
+ *         KStream<?, ?> output3();
+ *
+ *         ......
+ *
+ *     }
+ *
+ * + *
+ *     interface KStreamKtableProcessor {
+ *         @Input("input-1")
+ *         KStream<?, ?> input1();
+ *
+ *         @Input("input-2")
+ *         KTable<?, ?> input2();
+ *
+ *         @Output("output")
+ *         KStream<?, ?> output();
+ *
+ *         ......
+ *
+ *     }
+ *
+ * + * @author Marius Bogoevici + * @author Soby Chacko + */ +public interface KafkaStreamsProcessor { + + /** + * Input binding. + * @return {@link Input} binding for {@link KStream} type. + */ + @Input("input") + KStream input(); + + /** + * Output binding. + * @return {@link Output} binding for {@link KStream} type. + */ + @Output("output") + KStream output(); + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsStateStore.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsStateStore.java new file mode 100644 index 000000000..a24cda301 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/annotations/KafkaStreamsStateStore.java @@ -0,0 +1,115 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsStateStoreProperties; + +/** + * Interface for Kafka Stream state store. + * + * This interface can be used to inject a state store specification into KStream building + * process so that the desired store can be built by StreamBuilder and added to topology + * for later use by processors. This is particularly useful when need to combine stream + * DSL with low level processor APIs. In those cases, if a writable state store is desired + * in processors, it needs to be created using this annotation. Here is the example. + * + *
+ *     @StreamListener("input")
+ *     @KafkaStreamsStateStore(name="mystate", type= KafkaStreamsStateStoreProperties.StoreType.WINDOW,
+ *     								size=300000)
+ *	   public void process(KStream<Object, Product> input) {
+ *         ......
+ *     }
+ * 
+ * + * With that, you should be able to read/write this state store in your + * processor/transformer code. + * + *
+ * 		new Processor<Object, Product>() {
+ * 			WindowStore<Object, String> state;
+ * 			@Override
+ *			public void init(ProcessorContext processorContext) {
+ *			state = (WindowStore)processorContext.getStateStore("mystate");
+ *				......
+ *			}
+ *		}
+ * 
+ * + * @author Lei Chen + */ + +@Target({ ElementType.TYPE, ElementType.METHOD, ElementType.ANNOTATION_TYPE }) +@Retention(RetentionPolicy.RUNTIME) + +public @interface KafkaStreamsStateStore { + + /** + * Provides name of the state store. + * @return name of state store. + */ + String name() default ""; + + /** + * State store type. + * @return {@link KafkaStreamsStateStoreProperties.StoreType} of state store. + */ + KafkaStreamsStateStoreProperties.StoreType type() default KafkaStreamsStateStoreProperties.StoreType.KEYVALUE; + + /** + * Serde used for key. + * @return key serde of state store. + */ + String keySerde() default "org.apache.kafka.common.serialization.Serdes$StringSerde"; + + /** + * Serde used for value. + * @return value serde of state store. + */ + String valueSerde() default "org.apache.kafka.common.serialization.Serdes$StringSerde"; + + /** + * Length in milli-second of Windowed store window. + * @return length in milli-second of window(for windowed store). + */ + long lengthMs() default 0; + + /** + * Retention period for Windowed store windows. + * @return the maximum period of time in milli-second to keep each window in this + * store(for windowed store). + */ + long retentionMs() default 0; + + /** + * Whether catching is enabled or not. + * @return whether caching should be enabled on the created store. + */ + boolean cache() default false; + + /** + * Whether logging is enabled or not. + * @return whether logging should be enabled on the created store. + */ + boolean logging() default true; + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/endpoint/KafkaStreamsTopologyEndpoint.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/endpoint/KafkaStreamsTopologyEndpoint.java new file mode 100644 index 000000000..598a0165f --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/endpoint/KafkaStreamsTopologyEndpoint.java @@ -0,0 +1,73 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.endpoint; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.boot.actuate.endpoint.annotation.Endpoint; +import org.springframework.boot.actuate.endpoint.annotation.ReadOperation; +import org.springframework.boot.actuate.endpoint.annotation.Selector; +import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsRegistry; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.util.StringUtils; + +/** + * Actuator endpoint for topology description. + * + * @author Soby Chacko + * @since 3.0.4 + */ +@Endpoint(id = "kafkastreamstopology") +public class KafkaStreamsTopologyEndpoint { + + /** + * Topology not found message. + */ + public static final String NO_TOPOLOGY_FOUND_MSG = "No topology found for the given application ID"; + + private final KafkaStreamsRegistry kafkaStreamsRegistry; + + public KafkaStreamsTopologyEndpoint(KafkaStreamsRegistry kafkaStreamsRegistry) { + this.kafkaStreamsRegistry = kafkaStreamsRegistry; + } + + @ReadOperation + public List kafkaStreamsTopologies() { + final List streamsBuilderFactoryBeans = this.kafkaStreamsRegistry.streamsBuilderFactoryBeans(); + final StringBuilder topologyDescription = new StringBuilder(); + final List descs = new ArrayList<>(); + streamsBuilderFactoryBeans.stream() + .forEach(streamsBuilderFactoryBean -> + descs.add(streamsBuilderFactoryBean.getTopology().describe().toString())); + return descs; + } + + @ReadOperation + public String kafkaStreamsTopology(@Selector String applicationId) { + if (!StringUtils.isEmpty(applicationId)) { + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsRegistry.streamsBuilderFactoryBean(applicationId); + if (streamsBuilderFactoryBean != null) { + return streamsBuilderFactoryBean.getTopology().describe().toString(); + } + else { + return NO_TOPOLOGY_FOUND_MSG; + } + } + return NO_TOPOLOGY_FOUND_MSG; + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/endpoint/KafkaStreamsTopologyEndpointAutoConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/endpoint/KafkaStreamsTopologyEndpointAutoConfiguration.java new file mode 100644 index 000000000..a3c9f7763 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/endpoint/KafkaStreamsTopologyEndpointAutoConfiguration.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.endpoint; + +import org.springframework.boot.actuate.autoconfigure.endpoint.EndpointAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.endpoint.condition.ConditionalOnAvailableEndpoint; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsBinderSupportAutoConfiguration; +import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsRegistry; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author Soby Chacko + * @since 3.0.4 + */ +@Configuration +@ConditionalOnClass(name = { + "org.springframework.boot.actuate.endpoint.annotation.Endpoint" }) +@AutoConfigureAfter({EndpointAutoConfiguration.class, KafkaStreamsBinderSupportAutoConfiguration.class}) +public class KafkaStreamsTopologyEndpointAutoConfiguration { + + @Bean + @ConditionalOnAvailableEndpoint + public KafkaStreamsTopologyEndpoint topologyEndpoint(KafkaStreamsRegistry kafkaStreamsRegistry) { + return new KafkaStreamsTopologyEndpoint(kafkaStreamsRegistry); + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/FunctionDetectorCondition.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/FunctionDetectorCondition.java new file mode 100644 index 000000000..1f5890f83 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/FunctionDetectorCondition.java @@ -0,0 +1,140 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.function; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.streams.kstream.GlobalKTable; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; + +import org.springframework.beans.factory.BeanFactoryUtils; +import org.springframework.beans.factory.annotation.AnnotatedBeanDefinition; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.boot.autoconfigure.condition.ConditionOutcome; +import org.springframework.boot.autoconfigure.condition.SpringBootCondition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.ResolvableType; +import org.springframework.core.type.AnnotatedTypeMetadata; +import org.springframework.util.ClassUtils; +import org.springframework.util.CollectionUtils; + +/** + * Custom {@link org.springframework.context.annotation.Condition} that detects the presence + * of java.util.Function|Consumer beans. Used for Kafka Streams function support. + * + * @author Soby Chacko + * @since 2.2.0 + */ +public class FunctionDetectorCondition extends SpringBootCondition { + + private static final Log LOG = LogFactory.getLog(FunctionDetectorCondition.class); + + @SuppressWarnings({ "unchecked", "rawtypes" }) + @Override + public ConditionOutcome getMatchOutcome(ConditionContext context, AnnotatedTypeMetadata metadata) { + if (context != null && context.getBeanFactory() != null) { + + String[] functionTypes = BeanFactoryUtils.beanNamesForTypeIncludingAncestors(context.getBeanFactory(), Function.class, true, false); + String[] consumerTypes = BeanFactoryUtils.beanNamesForTypeIncludingAncestors(context.getBeanFactory(), Consumer.class, true, false); + String[] biFunctionTypes = BeanFactoryUtils.beanNamesForTypeIncludingAncestors(context.getBeanFactory(), BiFunction.class, true, false); + String[] biConsumerTypes = BeanFactoryUtils.beanNamesForTypeIncludingAncestors(context.getBeanFactory(), BiConsumer.class, true, false); + + List functionComponents = new ArrayList<>(); + + functionComponents.addAll(Arrays.asList(functionTypes)); + functionComponents.addAll(Arrays.asList(consumerTypes)); + functionComponents.addAll(Arrays.asList(biFunctionTypes)); + functionComponents.addAll(Arrays.asList(biConsumerTypes)); + + List kafkaStreamsFunctions = pruneFunctionBeansForKafkaStreams(functionComponents, context); + if (!CollectionUtils.isEmpty(kafkaStreamsFunctions)) { + return ConditionOutcome.match("Matched. Function/BiFunction/Consumer beans found"); + } + else { + return ConditionOutcome.noMatch("No match. No Function/BiFunction/Consumer beans found"); + } + } + return ConditionOutcome.noMatch("No match. No Function/BiFunction/Consumer beans found"); + } + + private static List pruneFunctionBeansForKafkaStreams(List functionComponents, + ConditionContext context) { + final List prunedList = new ArrayList<>(); + + for (String key : functionComponents) { + final Class classObj = ClassUtils.resolveClassName(((AnnotatedBeanDefinition) + context.getBeanFactory().getBeanDefinition(key)) + .getMetadata().getClassName(), + ClassUtils.getDefaultClassLoader()); + try { + + Method[] methods = classObj.getMethods(); + Optional kafkaStreamMethod = Arrays.stream(methods).filter(m -> m.getName().equals(key)).findFirst(); + // check if the bean name is overridden. + if (!kafkaStreamMethod.isPresent()) { + final BeanDefinition beanDefinition = context.getBeanFactory().getBeanDefinition(key); + final String factoryMethodName = beanDefinition.getFactoryMethodName(); + kafkaStreamMethod = Arrays.stream(methods).filter(m -> m.getName().equals(factoryMethodName)).findFirst(); + } + + if (kafkaStreamMethod.isPresent()) { + Method method = kafkaStreamMethod.get(); + ResolvableType resolvableType = ResolvableType.forMethodReturnType(method, classObj); + final Class rawClass = resolvableType.getGeneric(0).getRawClass(); + if (rawClass == KStream.class || rawClass == KTable.class || rawClass == GlobalKTable.class) { + prunedList.add(key); + } + } + else { + //check if it is a @Component bean. + Optional componentBeanMethod = Arrays.stream(methods).filter( + m -> (m.getName().equals("apply") || m.getName().equals("accept")) + && isKafkaStreamsTypeFound(m)).findFirst(); + if (componentBeanMethod.isPresent()) { + Method method = componentBeanMethod.get(); + final ResolvableType resolvableType1 = ResolvableType.forMethodParameter(method, 0); + final Class rawClass = resolvableType1.getRawClass(); + if (rawClass == KStream.class || rawClass == KTable.class || rawClass == GlobalKTable.class) { + prunedList.add(key); + } + } + } + } + catch (Exception e) { + LOG.error("Function not found: " + key, e); + } + } + return prunedList; + } + + private static boolean isKafkaStreamsTypeFound(Method method) { + return KStream.class.isAssignableFrom(method.getParameters()[0].getType()) || + KTable.class.isAssignableFrom(method.getParameters()[0].getType()) || + GlobalKTable.class.isAssignableFrom(method.getParameters()[0].getType()); + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsBindableProxyFactory.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsBindableProxyFactory.java new file mode 100644 index 000000000..80806e92b --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsBindableProxyFactory.java @@ -0,0 +1,286 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.function; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.apache.kafka.streams.kstream.GlobalKTable; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.BeanFactoryAware; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.beans.factory.support.RootBeanDefinition; +import org.springframework.cloud.stream.binding.AbstractBindableProxyFactory; +import org.springframework.cloud.stream.binding.BoundTargetHolder; +import org.springframework.cloud.stream.function.FunctionConstants; +import org.springframework.cloud.stream.function.StreamFunctionProperties; +import org.springframework.core.ResolvableType; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +/** + * Kafka Streams specific target bindings proxy factory. See {@link AbstractBindableProxyFactory} for more details. + *

+ * Targets bound by this factory: + *

+ * {@link KStream} + * {@link KTable} + * {@link GlobalKTable} + *

+ * This class looks at the Function bean's return signature as {@link ResolvableType} and introspect the individual types, + * binding them on the way. + *

+ * All types on the {@link ResolvableType} are bound except for KStream[] array types on the outbound, which will be + * deferred for binding at a later stage. The reason for doing that is because in this class, we don't have any way to know + * the actual size in the returned array. That has to wait until the function is invoked and we get a result. + * + * @author Soby Chacko + * @since 3.0.0 + */ +public class KafkaStreamsBindableProxyFactory extends AbstractBindableProxyFactory implements InitializingBean, BeanFactoryAware { + + @Autowired + private StreamFunctionProperties streamFunctionProperties; + + private ResolvableType[] types; + + private Method method; + + private final String functionName; + + private BeanFactory beanFactory; + + public KafkaStreamsBindableProxyFactory(ResolvableType[] types, String functionName, Method method) { + super(types[0].getType().getClass()); + this.types = types; + this.functionName = functionName; + this.method = method; + } + + @Override + public void afterPropertiesSet() { + populateBindingTargetFactories(beanFactory); + Assert.notEmpty(KafkaStreamsBindableProxyFactory.this.bindingTargetFactories, + "'bindingTargetFactories' cannot be empty"); + + int resolvableTypeDepthCounter = 0; + boolean isKafkaStreamsType = this.types[0].getRawClass().isAssignableFrom(KStream.class) || + this.types[0].getRawClass().isAssignableFrom(KTable.class) || + this.types[0].getRawClass().isAssignableFrom(GlobalKTable.class); + ResolvableType argument = isKafkaStreamsType ? this.types[0] : this.types[0].getGeneric(resolvableTypeDepthCounter++); + List inputBindings = buildInputBindings(); + Iterator iterator = inputBindings.iterator(); + String next = iterator.next(); + bindInput(argument, next); + + // Check if its a component style bean. + if (method != null) { + final Object bean = beanFactory.getBean(functionName); + if (BiFunction.class.isAssignableFrom(bean.getClass()) || BiConsumer.class.isAssignableFrom(bean.getClass())) { + argument = ResolvableType.forMethodParameter(method, 1); + next = iterator.next(); + bindInput(argument, next); + } + } + // Normal functional bean + if (this.types[0].getRawClass() != null && + (this.types[0].getRawClass().isAssignableFrom(BiFunction.class) || + this.types[0].getRawClass().isAssignableFrom(BiConsumer.class))) { + argument = this.types[0].getGeneric(resolvableTypeDepthCounter++); + next = iterator.next(); + bindInput(argument, next); + } + ResolvableType outboundArgument; + if (method != null) { + outboundArgument = ResolvableType.forMethodReturnType(method); + } + else { + outboundArgument = this.types[0].getGeneric(resolvableTypeDepthCounter); + } + + while (isAnotherFunctionOrConsumerFound(outboundArgument)) { + //The function is a curried function. We should introspect the partial function chain hierarchy. + argument = outboundArgument.getGeneric(0); + String next1 = iterator.next(); + bindInput(argument, next1); + outboundArgument = outboundArgument.getGeneric(1); + } + + + final int lastTypeIndex = this.types.length - 1; + if (this.types.length > 1 && this.types[lastTypeIndex] != null && this.types[lastTypeIndex].getRawClass() != null) { + if (this.types[lastTypeIndex].getRawClass().isAssignableFrom(Function.class) || + this.types[lastTypeIndex].getRawClass().isAssignableFrom(Consumer.class)) { + outboundArgument = this.types[lastTypeIndex].getGeneric(1); + } + } + + if (outboundArgument != null && outboundArgument.getRawClass() != null && (!outboundArgument.isArray() && + (outboundArgument.getRawClass().isAssignableFrom(KStream.class) || + outboundArgument.getRawClass().isAssignableFrom(KTable.class)))) { //Allowing both KStream and KTable on the outbound. + // if the type is array, we need to do a late binding as we don't know the number of + // output bindings at this point in the flow. + + List outputBindings = streamFunctionProperties.getOutputBindings(this.functionName); + String outputBinding = null; + + if (!CollectionUtils.isEmpty(outputBindings)) { + Iterator outputBindingsIter = outputBindings.iterator(); + if (outputBindingsIter.hasNext()) { + outputBinding = outputBindingsIter.next(); + } + } + else { + outputBinding = String.format("%s-%s-0", this.functionName, FunctionConstants.DEFAULT_OUTPUT_SUFFIX); + } + Assert.isTrue(outputBinding != null, "output binding is not inferred."); + // We will only allow KStream targets on the outbound. If the user provides a KTable, + // we still use the KStreamBinder to send it through the outbound. + // In that case before sending, we do a cast from KTable to KStream. + // See KafkaStreamsFunctionsProcessor#setupFunctionInvokerForKafkaStreams for details. + KafkaStreamsBindableProxyFactory.this.outputHolders.put(outputBinding, + new BoundTargetHolder(getBindingTargetFactory(KStream.class) + .createOutput(outputBinding), true)); + String outputBinding1 = outputBinding; + RootBeanDefinition rootBeanDefinition1 = new RootBeanDefinition(); + rootBeanDefinition1.setInstanceSupplier(() -> outputHolders.get(outputBinding1).getBoundTarget()); + BeanDefinitionRegistry registry = (BeanDefinitionRegistry) beanFactory; + registry.registerBeanDefinition(outputBinding1, rootBeanDefinition1); + } + } + + private boolean isAnotherFunctionOrConsumerFound(ResolvableType arg1) { + return arg1 != null && !arg1.isArray() && arg1.getRawClass() != null && + (arg1.getRawClass().isAssignableFrom(Function.class) || arg1.getRawClass().isAssignableFrom(Consumer.class)); + } + + /** + * If the application provides the property spring.cloud.stream.function.inputBindings.functionName, + * that gets precedence. Otherwise, use functionName-input or functionName-input-0, functionName-input-1 and so on + * for multiple inputs. + * + * @return an ordered collection of input bindings to use + */ + private List buildInputBindings() { + List inputs = new ArrayList<>(); + List inputBindings = streamFunctionProperties.getInputBindings(this.functionName); + if (!CollectionUtils.isEmpty(inputBindings)) { + inputs.addAll(inputBindings); + return inputs; + } + int numberOfInputs = this.types[0].getRawClass() != null && + (this.types[0].getRawClass().isAssignableFrom(BiFunction.class) || + this.types[0].getRawClass().isAssignableFrom(BiConsumer.class)) ? 2 : getNumberOfInputs(); + + // For @Component style beans. + if (method != null) { + final ResolvableType returnType = ResolvableType.forMethodReturnType(method); + Object bean = beanFactory.containsBean(functionName) ? beanFactory.getBean(functionName) : null; + + if (bean != null && (BiFunction.class.isAssignableFrom(bean.getClass()) || BiConsumer.class.isAssignableFrom(bean.getClass()))) { + numberOfInputs = 2; + } + else if (returnType.getRawClass().isAssignableFrom(Function.class) || returnType.getRawClass().isAssignableFrom(Consumer.class)) { + numberOfInputs = 1; + ResolvableType arg1 = returnType; + + while (isAnotherFunctionOrConsumerFound(arg1)) { + arg1 = arg1.getGeneric(1); + numberOfInputs++; + } + } + } + + int i = 0; + while (i < numberOfInputs) { + inputs.add(String.format("%s-%s-%d", this.functionName, FunctionConstants.DEFAULT_INPUT_SUFFIX, i++)); + } + return inputs; + } + + private int getNumberOfInputs() { + int numberOfInputs = 1; + ResolvableType arg1 = this.types[0].getGeneric(1); + + while (isAnotherFunctionOrConsumerFound(arg1)) { + arg1 = arg1.getGeneric(1); + numberOfInputs++; + } + return numberOfInputs; + } + + private void bindInput(ResolvableType arg0, String inputName) { + if (arg0.getRawClass() != null) { + KafkaStreamsBindableProxyFactory.this.inputHolders.put(inputName, + new BoundTargetHolder(getBindingTargetFactory(arg0.getRawClass()) + .createInput(inputName), true)); + } + BeanDefinitionRegistry registry = (BeanDefinitionRegistry) beanFactory; + RootBeanDefinition rootBeanDefinition = new RootBeanDefinition(); + rootBeanDefinition.setInstanceSupplier(() -> inputHolders.get(inputName).getBoundTarget()); + registry.registerBeanDefinition(inputName, rootBeanDefinition); + } + + @Override + public Set getInputs() { + Set ins = new LinkedHashSet<>(); + this.inputHolders.forEach((s, BoundTargetHolder) -> ins.add(s)); + return ins; + } + + @Override + public Set getOutputs() { + Set outs = new LinkedHashSet<>(); + this.outputHolders.forEach((s, BoundTargetHolder) -> outs.add(s)); + return outs; + } + + @Override + public void setBeanFactory(BeanFactory beanFactory) throws BeansException { + this.beanFactory = beanFactory; + } + + public void addOutputBinding(String output, Class clazz) { + KafkaStreamsBindableProxyFactory.this.outputHolders.put(output, + new BoundTargetHolder(getBindingTargetFactory(clazz) + .createOutput(output), true)); + } + + public String getFunctionName() { + return functionName; + } + + public Map getOutputHolders() { + return outputHolders; + } +} + diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionAutoConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionAutoConfiguration.java new file mode 100644 index 000000000..10588ccc7 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionAutoConfiguration.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.function; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsFunctionProcessor; +import org.springframework.cloud.stream.function.StreamFunctionProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; + +/** + * @author Soby Chacko + * @since 2.2.0 + */ +@Configuration +@EnableConfigurationProperties(StreamFunctionProperties.class) +public class KafkaStreamsFunctionAutoConfiguration { + + @Bean + @Conditional(FunctionDetectorCondition.class) + public KafkaStreamsFunctionProcessorInvoker kafkaStreamsFunctionProcessorInvoker( + KafkaStreamsFunctionBeanPostProcessor kafkaStreamsFunctionBeanPostProcessor, + KafkaStreamsFunctionProcessor kafkaStreamsFunctionProcessor, + KafkaStreamsBindableProxyFactory[] kafkaStreamsBindableProxyFactories, + StreamFunctionProperties streamFunctionProperties) { + return new KafkaStreamsFunctionProcessorInvoker(kafkaStreamsFunctionBeanPostProcessor.getResolvableTypes(), + kafkaStreamsFunctionProcessor, kafkaStreamsBindableProxyFactories, kafkaStreamsFunctionBeanPostProcessor.getMethods(), + streamFunctionProperties); + } + + @Bean + @Conditional(FunctionDetectorCondition.class) + public KafkaStreamsFunctionBeanPostProcessor kafkaStreamsFunctionBeanPostProcessor(StreamFunctionProperties streamFunctionProperties) { + return new KafkaStreamsFunctionBeanPostProcessor(streamFunctionProperties); + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionBeanPostProcessor.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionBeanPostProcessor.java new file mode 100644 index 000000000..153f19d42 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionBeanPostProcessor.java @@ -0,0 +1,273 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.function; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.streams.kstream.GlobalKTable; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.BeanFactoryAware; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.AnnotatedBeanDefinition; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.beans.factory.support.RootBeanDefinition; +import org.springframework.cloud.stream.function.StreamFunctionProperties; +import org.springframework.core.ResolvableType; +import org.springframework.util.ClassUtils; +import org.springframework.util.StringUtils; + +/** + * @author Soby Chacko + * @since 2.2.0 + */ +public class KafkaStreamsFunctionBeanPostProcessor implements InitializingBean, BeanFactoryAware { + + private static final Log LOG = LogFactory.getLog(KafkaStreamsFunctionBeanPostProcessor.class); + + private static final String[] EXCLUDE_FUNCTIONS = new String[]{"functionRouter", "sendToDlqAndContinue"}; + + private ConfigurableListableBeanFactory beanFactory; + private boolean onlySingleFunction; + private Map resolvableTypeMap = new TreeMap<>(); + private Map methods = new TreeMap<>(); + + private final StreamFunctionProperties streamFunctionProperties; + + private Map kafkaStreamsOnlyResolvableTypes = new HashMap<>(); + private Map kafakStreamsOnlyMethods = new HashMap<>(); + + public KafkaStreamsFunctionBeanPostProcessor(StreamFunctionProperties streamFunctionProperties) { + this.streamFunctionProperties = streamFunctionProperties; + } + + public Map getResolvableTypes() { + return this.resolvableTypeMap; + } + + public Map getMethods() { + return methods; + } + + @Override + public void afterPropertiesSet() { + String[] functionNames = this.beanFactory.getBeanNamesForType(Function.class); + String[] biFunctionNames = this.beanFactory.getBeanNamesForType(BiFunction.class); + String[] consumerNames = this.beanFactory.getBeanNamesForType(Consumer.class); + String[] biConsumerNames = this.beanFactory.getBeanNamesForType(BiConsumer.class); + + final Stream concat = Stream.concat( + Stream.concat(Stream.of(functionNames), Stream.of(consumerNames)), + Stream.concat(Stream.of(biFunctionNames), Stream.of(biConsumerNames))); + final List collect = concat.collect(Collectors.toList()); + collect.removeIf(s -> Arrays.stream(EXCLUDE_FUNCTIONS).anyMatch(t -> t.equals(s))); + collect.removeIf(Pattern.compile(".*_registration").asPredicate()); + + onlySingleFunction = collect.size() == 1; + collect.stream() + .forEach(this::extractResolvableTypes); + + kafkaStreamsOnlyResolvableTypes.keySet().forEach(k -> addResolvableTypeInfo(k, kafkaStreamsOnlyResolvableTypes.get(k))); + kafakStreamsOnlyMethods.keySet().forEach(k -> addResolvableTypeInfo(k, kafakStreamsOnlyMethods.get(k))); + + BeanDefinitionRegistry registry = (BeanDefinitionRegistry) beanFactory; + + final String definition = streamFunctionProperties.getDefinition(); + final String[] functionUnits = StringUtils.hasText(definition) ? definition.split(";") : new String[]{}; + + final Set kafkaStreamsMethodNames = new HashSet<>(kafkaStreamsOnlyResolvableTypes.keySet()); + kafkaStreamsMethodNames.addAll(this.resolvableTypeMap.keySet()); + + if (functionUnits.length == 0) { + for (String s : getResolvableTypes().keySet()) { + ResolvableType[] resolvableTypes = new ResolvableType[]{getResolvableTypes().get(s)}; + RootBeanDefinition rootBeanDefinition = new RootBeanDefinition( + KafkaStreamsBindableProxyFactory.class); + registerKakaStreamsProxyFactory(registry, s, resolvableTypes, rootBeanDefinition); + } + } + else { + for (String functionUnit : functionUnits) { + if (functionUnit.contains("|")) { + final String[] composedFunctions = functionUnit.split("\\|"); + String derivedNameFromComposed = ""; + ResolvableType[] resolvableTypes = new ResolvableType[composedFunctions.length]; + + int i = 0; + boolean nonKafkaStreamsFunctionsFound = false; + + for (String split : composedFunctions) { + derivedNameFromComposed = derivedNameFromComposed.concat(split); + resolvableTypes[i++] = getResolvableTypes().get(split); + if (!kafkaStreamsMethodNames.contains(split)) { + nonKafkaStreamsFunctionsFound = true; + break; + } + } + if (!nonKafkaStreamsFunctionsFound) { + RootBeanDefinition rootBeanDefinition = new RootBeanDefinition( + KafkaStreamsBindableProxyFactory.class); + registerKakaStreamsProxyFactory(registry, derivedNameFromComposed, resolvableTypes, rootBeanDefinition); + } + } + else { + // Ensure that the function unit is a Kafka Streams function + if (kafkaStreamsMethodNames.contains(functionUnit)) { + ResolvableType[] resolvableTypes = new ResolvableType[]{getResolvableTypes().get(functionUnit)}; + RootBeanDefinition rootBeanDefinition = new RootBeanDefinition( + KafkaStreamsBindableProxyFactory.class); + registerKakaStreamsProxyFactory(registry, functionUnit, resolvableTypes, rootBeanDefinition); + } + } + } + } + } + + private void registerKakaStreamsProxyFactory(BeanDefinitionRegistry registry, String s, ResolvableType[] resolvableTypes, RootBeanDefinition rootBeanDefinition) { + rootBeanDefinition.getConstructorArgumentValues() + .addGenericArgumentValue(resolvableTypes); + rootBeanDefinition.getConstructorArgumentValues() + .addGenericArgumentValue(s); + rootBeanDefinition.getConstructorArgumentValues() + .addGenericArgumentValue(getMethods().get(s)); + registry.registerBeanDefinition("kafkaStreamsBindableProxyFactory-" + s, rootBeanDefinition); + } + + private void extractResolvableTypes(String key) { + final Class classObj = ClassUtils.resolveClassName(((AnnotatedBeanDefinition) + this.beanFactory.getBeanDefinition(key)) + .getMetadata().getClassName(), + ClassUtils.getDefaultClassLoader()); + try { + Method[] methods = classObj.getMethods(); + Optional functionalBeanMethods = Arrays.stream(methods).filter(m -> m.getName().equals(key)).findFirst(); + if (!functionalBeanMethods.isPresent()) { + final BeanDefinition beanDefinition = this.beanFactory.getBeanDefinition(key); + final String factoryMethodName = beanDefinition.getFactoryMethodName(); + functionalBeanMethods = Arrays.stream(methods).filter(m -> m.getName().equals(factoryMethodName)).findFirst(); + } + + if (functionalBeanMethods.isPresent()) { + Method method = functionalBeanMethods.get(); + ResolvableType resolvableType = ResolvableType.forMethodReturnType(method, classObj); + final Class rawClass = resolvableType.getGeneric(0).getRawClass(); + if (rawClass == KStream.class || rawClass == KTable.class || rawClass == GlobalKTable.class) { + if (onlySingleFunction) { + resolvableTypeMap.put(key, resolvableType); + } + else { + discoverOnlyKafkaStreamsResolvableTypes(key, resolvableType); + } + } + } + else { + Optional componentBeanMethods = Arrays.stream(methods) + .filter(m -> m.getName().equals("apply") && isKafkaStreamsTypeFound(m) || + m.getName().equals("accept") && isKafkaStreamsTypeFound(m)).findFirst(); + if (componentBeanMethods.isPresent()) { + Method method = componentBeanMethods.get(); + final ResolvableType resolvableType = ResolvableType.forMethodParameter(method, 0); + final Class rawClass = resolvableType.getRawClass(); + if (rawClass == KStream.class || rawClass == KTable.class || rawClass == GlobalKTable.class) { + if (onlySingleFunction) { + resolvableTypeMap.put(key, resolvableType); + this.methods.put(key, method); + } + else { + discoverOnlyKafkaStreamsResolvableTypesAndMethods(key, resolvableType, method); + } + } + } + } + } + catch (Exception e) { + LOG.error("Function activation issues while mapping the function: " + key, e); + } + } + + private void addResolvableTypeInfo(String key, ResolvableType resolvableType) { + if (kafkaStreamsOnlyResolvableTypes.size() == 1) { + resolvableTypeMap.put(key, resolvableType); + } + else { + final String definition = streamFunctionProperties.getDefinition(); + if (definition == null) { + throw new IllegalStateException("Multiple functions found, but function definition property is not set."); + } + else if (definition.contains(key)) { + resolvableTypeMap.put(key, resolvableType); + } + } + } + + private void discoverOnlyKafkaStreamsResolvableTypes(String key, ResolvableType resolvableType) { + kafkaStreamsOnlyResolvableTypes.put(key, resolvableType); + } + + private void discoverOnlyKafkaStreamsResolvableTypesAndMethods(String key, ResolvableType resolvableType, Method method) { + kafkaStreamsOnlyResolvableTypes.put(key, resolvableType); + kafakStreamsOnlyMethods.put(key, method); + } + + private void addResolvableTypeInfo(String key, Method method) { + if (kafakStreamsOnlyMethods.size() == 1) { + this.methods.put(key, method); + } + else { + final String definition = streamFunctionProperties.getDefinition(); + if (definition == null) { + throw new IllegalStateException("Multiple functions found, but function definition property is not set."); + } + else if (definition.contains(key)) { + this.methods.put(key, method); + } + } + } + + private boolean isKafkaStreamsTypeFound(Method method) { + return KStream.class.isAssignableFrom(method.getParameters()[0].getType()) || + KTable.class.isAssignableFrom(method.getParameters()[0].getType()) || + GlobalKTable.class.isAssignableFrom(method.getParameters()[0].getType()); + } + + @Override + public void setBeanFactory(BeanFactory beanFactory) throws BeansException { + this.beanFactory = (ConfigurableListableBeanFactory) beanFactory; + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionProcessorInvoker.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionProcessorInvoker.java new file mode 100644 index 000000000..aa26f82e8 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionProcessorInvoker.java @@ -0,0 +1,89 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.function; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; + +import javax.annotation.PostConstruct; + +import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsFunctionProcessor; +import org.springframework.cloud.stream.function.StreamFunctionProperties; +import org.springframework.core.ResolvableType; +import org.springframework.util.StringUtils; + +/** + * @author Soby Chacko + * @since 2.1.0 + */ +public class KafkaStreamsFunctionProcessorInvoker { + + private final KafkaStreamsFunctionProcessor kafkaStreamsFunctionProcessor; + private final Map resolvableTypeMap; + private final KafkaStreamsBindableProxyFactory[] kafkaStreamsBindableProxyFactories; + private final Map methods; + private final StreamFunctionProperties streamFunctionProperties; + + public KafkaStreamsFunctionProcessorInvoker(Map resolvableTypeMap, + KafkaStreamsFunctionProcessor kafkaStreamsFunctionProcessor, + KafkaStreamsBindableProxyFactory[] kafkaStreamsBindableProxyFactories, + Map methods, StreamFunctionProperties streamFunctionProperties) { + this.kafkaStreamsFunctionProcessor = kafkaStreamsFunctionProcessor; + this.resolvableTypeMap = resolvableTypeMap; + this.kafkaStreamsBindableProxyFactories = kafkaStreamsBindableProxyFactories; + this.methods = methods; + this.streamFunctionProperties = streamFunctionProperties; + } + + @PostConstruct + void invoke() { + final String definition = streamFunctionProperties.getDefinition(); + final String[] functionUnits = StringUtils.hasText(definition) ? definition.split(";") : new String[]{}; + + if (functionUnits.length == 0) { + resolvableTypeMap.forEach((key, value) -> { + Optional proxyFactory = + Arrays.stream(kafkaStreamsBindableProxyFactories).filter(p -> p.getFunctionName().equals(key)).findFirst(); + this.kafkaStreamsFunctionProcessor.setupFunctionInvokerForKafkaStreams(value, key, proxyFactory.get(), methods.get(key), null); + }); + } + + for (String functionUnit : functionUnits) { + if (functionUnit.contains("|")) { + final String[] composedFunctions = functionUnit.split("\\|"); + String[] derivedNameFromComposed = new String[]{""}; + for (String split : composedFunctions) { + derivedNameFromComposed[0] = derivedNameFromComposed[0].concat(split); + } + Optional proxyFactory = + Arrays.stream(kafkaStreamsBindableProxyFactories).filter(p -> p.getFunctionName().equals(derivedNameFromComposed[0])).findFirst(); + proxyFactory.ifPresent(kafkaStreamsBindableProxyFactory -> + this.kafkaStreamsFunctionProcessor.setupFunctionInvokerForKafkaStreams(resolvableTypeMap.get(composedFunctions[0]), + derivedNameFromComposed[0], kafkaStreamsBindableProxyFactory, methods.get(derivedNameFromComposed[0]), resolvableTypeMap.get(composedFunctions[composedFunctions.length - 1]), composedFunctions)); + } + else { + Optional proxyFactory = + Arrays.stream(kafkaStreamsBindableProxyFactories).filter(p -> p.getFunctionName().equals(functionUnit)).findFirst(); + proxyFactory.ifPresent(kafkaStreamsBindableProxyFactory -> + this.kafkaStreamsFunctionProcessor.setupFunctionInvokerForKafkaStreams(resolvableTypeMap.get(functionUnit), functionUnit, + kafkaStreamsBindableProxyFactory, methods.get(functionUnit), null)); + } + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsBinderConfigurationProperties.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsBinderConfigurationProperties.java new file mode 100644 index 000000000..95b1c93bc --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsBinderConfigurationProperties.java @@ -0,0 +1,191 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.properties; + +import java.util.HashMap; +import java.util.Map; + +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.streams.DeserializationExceptionHandler; + +/** + * Kafka Streams binder configuration properties. + * + * @author Soby Chacko + * @author Gary Russell + */ +public class KafkaStreamsBinderConfigurationProperties + extends KafkaBinderConfigurationProperties { + + public KafkaStreamsBinderConfigurationProperties(KafkaProperties kafkaProperties) { + super(kafkaProperties); + } + + /** + * Enumeration for various Serde errors. + * + * @deprecated in favor of {@link DeserializationExceptionHandler}. + */ + @Deprecated + public enum SerdeError { + + /** + * Deserialization error handler with log and continue. + */ + logAndContinue, + /** + * Deserialization error handler with log and fail. + */ + logAndFail, + /** + * Deserialization error handler with DLQ send. + */ + sendToDlq + + } + + private String applicationId; + + private StateStoreRetry stateStoreRetry = new StateStoreRetry(); + + private Map functions = new HashMap<>(); + + private KafkaStreamsBinderConfigurationProperties.SerdeError serdeError; + + /** + * {@link org.apache.kafka.streams.errors.DeserializationExceptionHandler} to use when + * there is a deserialization exception. This handler will be applied against all input bindings + * unless overridden at the consumer binding. + */ + private DeserializationExceptionHandler deserializationExceptionHandler; + + private boolean includeStoppedProcessorsForHealthCheck; + + public Map getFunctions() { + return functions; + } + + public void setFunctions(Map functions) { + this.functions = functions; + } + + public StateStoreRetry getStateStoreRetry() { + return stateStoreRetry; + } + + public void setStateStoreRetry(StateStoreRetry stateStoreRetry) { + this.stateStoreRetry = stateStoreRetry; + } + + public String getApplicationId() { + return this.applicationId; + } + + public void setApplicationId(String applicationId) { + this.applicationId = applicationId; + } + + @Deprecated + public KafkaStreamsBinderConfigurationProperties.SerdeError getSerdeError() { + return this.serdeError; + } + + @Deprecated + public void setSerdeError( + KafkaStreamsBinderConfigurationProperties.SerdeError serdeError) { + this.serdeError = serdeError; + if (serdeError == SerdeError.logAndContinue) { + this.deserializationExceptionHandler = DeserializationExceptionHandler.logAndContinue; + } + else if (serdeError == SerdeError.logAndFail) { + this.deserializationExceptionHandler = DeserializationExceptionHandler.logAndFail; + } + else if (serdeError == SerdeError.sendToDlq) { + this.deserializationExceptionHandler = DeserializationExceptionHandler.sendToDlq; + } + } + + public DeserializationExceptionHandler getDeserializationExceptionHandler() { + return deserializationExceptionHandler; + } + + public void setDeserializationExceptionHandler(DeserializationExceptionHandler deserializationExceptionHandler) { + this.deserializationExceptionHandler = deserializationExceptionHandler; + } + + public boolean isIncludeStoppedProcessorsForHealthCheck() { + return includeStoppedProcessorsForHealthCheck; + } + + public void setIncludeStoppedProcessorsForHealthCheck(boolean includeStoppedProcessorsForHealthCheck) { + this.includeStoppedProcessorsForHealthCheck = includeStoppedProcessorsForHealthCheck; + } + + public static class StateStoreRetry { + + private int maxAttempts = 1; + + private long backoffPeriod = 1000; + + public int getMaxAttempts() { + return maxAttempts; + } + + public void setMaxAttempts(int maxAttempts) { + this.maxAttempts = maxAttempts; + } + + public long getBackoffPeriod() { + return backoffPeriod; + } + + public void setBackoffPeriod(long backoffPeriod) { + this.backoffPeriod = backoffPeriod; + } + } + + public static class Functions { + + /** + * Function specific application id. + */ + private String applicationId; + + /** + * Funcion specific configuraiton to use. + */ + private Map configuration; + + public String getApplicationId() { + return applicationId; + } + + public void setApplicationId(String applicationId) { + this.applicationId = applicationId; + } + + public Map getConfiguration() { + return configuration; + } + + public void setConfiguration(Map configuration) { + this.configuration = configuration; + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsBindingProperties.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsBindingProperties.java new file mode 100644 index 000000000..dd61732ca --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsBindingProperties.java @@ -0,0 +1,49 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.properties; + +import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider; + +/** + * Extended binding properties holder that delegates to Kafka Streams producer and + * consumer properties. + * + * @author Marius Bogoevici + */ +public class KafkaStreamsBindingProperties implements BinderSpecificPropertiesProvider { + + private KafkaStreamsConsumerProperties consumer = new KafkaStreamsConsumerProperties(); + + private KafkaStreamsProducerProperties producer = new KafkaStreamsProducerProperties(); + + public KafkaStreamsConsumerProperties getConsumer() { + return this.consumer; + } + + public void setConsumer(KafkaStreamsConsumerProperties consumer) { + this.consumer = consumer; + } + + public KafkaStreamsProducerProperties getProducer() { + return this.producer; + } + + public void setProducer(KafkaStreamsProducerProperties producer) { + this.producer = producer; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsConsumerProperties.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsConsumerProperties.java new file mode 100644 index 000000000..17b10a962 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsConsumerProperties.java @@ -0,0 +1,145 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.properties; + +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.DeserializationExceptionHandler; + +/** + * Extended properties for Kafka Streams consumer. + * + * @author Marius Bogoevici + * @author Soby Chacko + */ +public class KafkaStreamsConsumerProperties extends KafkaConsumerProperties { + + private String applicationId; + + /** + * Key serde specified per binding. + */ + private String keySerde; + + /** + * Value serde specified per binding. + */ + private String valueSerde; + + /** + * Materialized as a KeyValueStore. + */ + private String materializedAs; + + /** + * Per input binding deserialization handler. + */ + private DeserializationExceptionHandler deserializationExceptionHandler; + + /** + * {@link org.apache.kafka.streams.processor.TimestampExtractor} bean name to use for this consumer. + */ + private String timestampExtractorBeanName; + + /** + * Comma separated list of supported event types for this binding. + */ + private String eventTypes; + + /** + * Record level header key for event type. + * If the default value is overridden, then that is expected on each record header if eventType based + * routing is enabled on this binding (by setting eventTypes). + */ + private String eventTypeHeaderKey = "event_type"; + + /** + * Custom name for the source component from which the processor is consuming from. + */ + private String consumedAs; + + public String getApplicationId() { + return this.applicationId; + } + + public void setApplicationId(String applicationId) { + this.applicationId = applicationId; + } + + public String getKeySerde() { + return this.keySerde; + } + + public void setKeySerde(String keySerde) { + this.keySerde = keySerde; + } + + public String getValueSerde() { + return this.valueSerde; + } + + public void setValueSerde(String valueSerde) { + this.valueSerde = valueSerde; + } + + public String getMaterializedAs() { + return this.materializedAs; + } + + public void setMaterializedAs(String materializedAs) { + this.materializedAs = materializedAs; + } + + public String getTimestampExtractorBeanName() { + return timestampExtractorBeanName; + } + + public void setTimestampExtractorBeanName(String timestampExtractorBeanName) { + this.timestampExtractorBeanName = timestampExtractorBeanName; + } + + public DeserializationExceptionHandler getDeserializationExceptionHandler() { + return deserializationExceptionHandler; + } + + public void setDeserializationExceptionHandler(DeserializationExceptionHandler deserializationExceptionHandler) { + this.deserializationExceptionHandler = deserializationExceptionHandler; + } + + public String getEventTypes() { + return eventTypes; + } + + public void setEventTypes(String eventTypes) { + this.eventTypes = eventTypes; + } + + public String getEventTypeHeaderKey() { + return this.eventTypeHeaderKey; + } + + public void setEventTypeHeaderKey(String eventTypeHeaderKey) { + this.eventTypeHeaderKey = eventTypeHeaderKey; + } + + public String getConsumedAs() { + return consumedAs; + } + + public void setConsumedAs(String consumedAs) { + this.consumedAs = consumedAs; + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsExtendedBindingProperties.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsExtendedBindingProperties.java new file mode 100644 index 000000000..4086748b0 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsExtendedBindingProperties.java @@ -0,0 +1,56 @@ +/* + * Copyright 2017-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.properties; + +import java.util.Map; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.cloud.stream.binder.AbstractExtendedBindingProperties; +import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider; + +/** + * Kafka streams specific extended binding properties class that extends from + * {@link AbstractExtendedBindingProperties}. + * + * @author Marius Bogoevici + * @author Oleg Zhurakousky + */ +@ConfigurationProperties("spring.cloud.stream.kafka.streams") +public class KafkaStreamsExtendedBindingProperties + // @checkstyle:off + extends + AbstractExtendedBindingProperties { + + // @checkstyle:on + private static final String DEFAULTS_PREFIX = "spring.cloud.stream.kafka.streams.default"; + + @Override + public String getDefaultsPrefix() { + return DEFAULTS_PREFIX; + } + + @Override + public Map getBindings() { + return this.doGetBindings(); + } + + @Override + public Class getExtendedPropertiesEntryClass() { + return KafkaStreamsBindingProperties.class; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsProducerProperties.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsProducerProperties.java new file mode 100644 index 000000000..152621c5e --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsProducerProperties.java @@ -0,0 +1,80 @@ +/* + * Copyright 2017-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.properties; + +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; + +/** + * Extended properties for Kafka Streams producer. + * + * @author Marius Bogoevici + * @author Soby Chacko + */ +public class KafkaStreamsProducerProperties extends KafkaProducerProperties { + + /** + * Key serde specified per binding. + */ + private String keySerde; + + /** + * Value serde specified per binding. + */ + private String valueSerde; + + /** + * {@link org.apache.kafka.streams.processor.StreamPartitioner} to be used on Kafka Streams producer. + */ + private String streamPartitionerBeanName; + + /** + * Custom name for the sink component to which the processor is producing to. + */ + private String producedAs; + + public String getKeySerde() { + return this.keySerde; + } + + public void setKeySerde(String keySerde) { + this.keySerde = keySerde; + } + + public String getValueSerde() { + return this.valueSerde; + } + + public void setValueSerde(String valueSerde) { + this.valueSerde = valueSerde; + } + + public String getStreamPartitionerBeanName() { + return this.streamPartitionerBeanName; + } + + public void setStreamPartitionerBeanName(String streamPartitionerBeanName) { + this.streamPartitionerBeanName = streamPartitionerBeanName; + } + + public String getProducedAs() { + return producedAs; + } + + public void setProducedAs(String producedAs) { + this.producedAs = producedAs; + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsStateStoreProperties.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsStateStoreProperties.java new file mode 100644 index 000000000..c51cc6120 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsStateStoreProperties.java @@ -0,0 +1,161 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.properties; + +/** + * Properties for Kafka Streams state store. + * + * @author Lei Chen + */ +public class KafkaStreamsStateStoreProperties { + + /** + * Enumeration for store type. + */ + public enum StoreType { + + /** + * Key value store. + */ + KEYVALUE("keyvalue"), + /** + * Window store. + */ + WINDOW("window"), + /** + * Session store. + */ + SESSION("session"); + + private final String type; + + StoreType(final String type) { + this.type = type; + } + + @Override + public String toString() { + return this.type; + } + + } + + /** + * Name for this state store. + */ + private String name; + + /** + * Type for this state store. + */ + private StoreType type; + + /** + * Size/length of this state store in ms. Only applicable for window store. + */ + private long length; + + /** + * Retention period for this state store in ms. + */ + private long retention; + + /** + * Key serde class specified per state store. + */ + private String keySerdeString; + + /** + * Value serde class specified per state store. + */ + private String valueSerdeString; + + /** + * Whether caching is enabled on this state store. + */ + private boolean cacheEnabled; + + /** + * Whether logging is enabled on this state store. + */ + private boolean loggingDisabled; + + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + + public StoreType getType() { + return this.type; + } + + public void setType(StoreType type) { + this.type = type; + } + + public long getLength() { + return this.length; + } + + public void setLength(long length) { + this.length = length; + } + + public long getRetention() { + return this.retention; + } + + public void setRetention(long retention) { + this.retention = retention; + } + + public String getKeySerdeString() { + return this.keySerdeString; + } + + public void setKeySerdeString(String keySerdeString) { + this.keySerdeString = keySerdeString; + } + + public String getValueSerdeString() { + return this.valueSerdeString; + } + + public void setValueSerdeString(String valueSerdeString) { + this.valueSerdeString = valueSerdeString; + } + + public boolean isCacheEnabled() { + return this.cacheEnabled; + } + + public void setCacheEnabled(boolean cacheEnabled) { + this.cacheEnabled = cacheEnabled; + } + + public boolean isLoggingDisabled() { + return this.loggingDisabled; + } + + public void setLoggingDisabled(boolean loggingDisabled) { + this.loggingDisabled = loggingDisabled; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CollectionSerde.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CollectionSerde.java new file mode 100644 index 000000000..ce217baf7 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CollectionSerde.java @@ -0,0 +1,245 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.serde; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.Map; +import java.util.PriorityQueue; + +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.Serializer; + +import org.springframework.kafka.support.serializer.JsonSerde; + +/** + * A convenient {@link Serde} for {@link java.util.Collection} implementations. + * + * Whenever a Kafka Stream application needs to collect data into a container object like + * {@link java.util.Collection}, then this Serde class can be used as a convenience for + * serialization needs. Some examples of where using this may handy is when the application + * needs to do aggregation or reduction operations where it needs to simply hold an + * {@link Iterable} type. + * + * By default, this Serde will use {@link JsonSerde} for serializing the inner objects. + * This can be changed by providing an explicit Serde during creation of this object. + * + * Here is an example of a possible use case: + * + *

+ *		.aggregate(ArrayList::new,
+ * 					(k, v, aggregates) -> {
+ * 							aggregates.add(v);
+ * 							return aggregates;
+ *                                                },
+ * 					Materialized.<String, Collection<Foo>, WindowStore<Bytes, byte[]>>as(
+ * 						"foo-store")
+ * 						.withKeySerde(Serdes.String())
+ *						.withValueSerde(new CollectionSerde<>(Foo.class, ArrayList.class)))
+ *  * 
+ * + * Supported Collection types by this Serde are - {@link java.util.ArrayList}, {@link java.util.LinkedList}, + * {@link java.util.PriorityQueue} and {@link java.util.HashSet}. Deserializer will throw an exception + * if any other Collection types are used. + * + * @param type of the underlying object that the collection holds + * @author Soby Chacko + * @since 3.0.0 + */ +public class CollectionSerde implements Serde> { + + /** + * Serde used for serializing the inner object. + */ + private final Serde> inner; + + /** + * Type of the collection class. This has to be a class that is + * implementing the {@link java.util.Collection} interface. + */ + private final Class collectionClass; + + /** + * Constructor to use when the application wants to specify the type + * of the Serde used for the inner object. + * + * @param serde specify an explicit Serde + * @param collectionsClass type of the Collection class + */ + public CollectionSerde(Serde serde, Class collectionsClass) { + this.collectionClass = collectionsClass; + this.inner = + Serdes.serdeFrom( + new CollectionSerializer<>(serde.serializer()), + new CollectionDeserializer<>(serde.deserializer(), collectionsClass)); + } + + /** + * Constructor to delegate serialization operations for the inner objects + * to {@link JsonSerde}. + * + * @param targetTypeForJsonSerde target type used by the JsonSerde + * @param collectionsClass type of the Collection class + */ + public CollectionSerde(Class targetTypeForJsonSerde, Class collectionsClass) { + this.collectionClass = collectionsClass; + try (JsonSerde jsonSerde = new JsonSerde(targetTypeForJsonSerde)) { + + this.inner = Serdes.serdeFrom( + new CollectionSerializer<>(jsonSerde.serializer()), + new CollectionDeserializer<>(jsonSerde.deserializer(), collectionsClass)); + } + } + + @Override + public Serializer> serializer() { + return inner.serializer(); + } + + @Override + public Deserializer> deserializer() { + return inner.deserializer(); + } + + @Override + public void configure(Map configs, boolean isKey) { + inner.serializer().configure(configs, isKey); + inner.deserializer().configure(configs, isKey); + } + + @Override + public void close() { + inner.serializer().close(); + inner.deserializer().close(); + } + + private static class CollectionSerializer implements Serializer> { + + + private Serializer inner; + + CollectionSerializer(Serializer inner) { + this.inner = inner; + } + + CollectionSerializer() { } + + @Override + public void configure(Map configs, boolean isKey) { + + } + + @Override + public byte[] serialize(String topic, Collection collection) { + final int size = collection.size(); + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final DataOutputStream dos = new DataOutputStream(baos); + final Iterator iterator = collection.iterator(); + try { + dos.writeInt(size); + while (iterator.hasNext()) { + final byte[] bytes = inner.serialize(topic, iterator.next()); + dos.writeInt(bytes.length); + dos.write(bytes); + } + } + catch (IOException e) { + throw new RuntimeException("Unable to serialize the provided collection", e); + } + return baos.toByteArray(); + } + + @Override + public void close() { + inner.close(); + } + } + + private static class CollectionDeserializer implements Deserializer> { + private final Deserializer valueDeserializer; + private final Class collectionClass; + + CollectionDeserializer(final Deserializer valueDeserializer, Class collectionClass) { + this.valueDeserializer = valueDeserializer; + this.collectionClass = collectionClass; + } + + @Override + public void configure(Map configs, boolean isKey) { + } + + @Override + public Collection deserialize(String topic, byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return null; + } + + Collection collection = getCollection(); + final DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + + try { + final int records = dataInputStream.readInt(); + for (int i = 0; i < records; i++) { + final byte[] valueBytes = new byte[dataInputStream.readInt()]; + final int read = dataInputStream.read(valueBytes); + if (read != -1) { + collection.add(valueDeserializer.deserialize(topic, valueBytes)); + } + } + } + catch (IOException e) { + throw new RuntimeException("Unable to deserialize collection", e); + } + + return collection; + } + + @Override + public void close() { + } + + private Collection getCollection() { + Collection collection; + if (this.collectionClass.isAssignableFrom(ArrayList.class)) { + collection = new ArrayList<>(); + } + else if (this.collectionClass.isAssignableFrom(HashSet.class)) { + collection = new HashSet<>(); + } + else if (this.collectionClass.isAssignableFrom(LinkedList.class)) { + collection = new LinkedList<>(); + } + else if (this.collectionClass.isAssignableFrom(PriorityQueue.class)) { + collection = new PriorityQueue<>(); + } + else { + throw new IllegalArgumentException("Unsupported collection type - " + this.collectionClass); + } + return collection; + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CompositeNonNativeSerde.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CompositeNonNativeSerde.java new file mode 100644 index 000000000..a7b0300ac --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CompositeNonNativeSerde.java @@ -0,0 +1,37 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.serde; + +import org.springframework.messaging.converter.CompositeMessageConverter; + +/** + * This class provides the same functionality as {@link MessageConverterDelegateSerde} and is deprecated. + * It is kept for backward compatibility reasons and will be removed in version 3.1 + * + * @author Soby Chacko + * @since 2.1 + * + * @deprecated in favor of {@link MessageConverterDelegateSerde} + */ +@Deprecated +public class CompositeNonNativeSerde extends MessageConverterDelegateSerde { + + public CompositeNonNativeSerde(CompositeMessageConverter compositeMessageConverter) { + super(compositeMessageConverter); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/MessageConverterDelegateSerde.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/MessageConverterDelegateSerde.java new file mode 100644 index 000000000..a64e28695 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/serde/MessageConverterDelegateSerde.java @@ -0,0 +1,228 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.serde; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serializer; + +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.converter.CompositeMessageConverter; +import org.springframework.messaging.converter.MessageConverter; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.util.Assert; +import org.springframework.util.MimeType; +import org.springframework.util.MimeTypeUtils; + +/** + * A {@link Serde} implementation that wraps the list of {@link MessageConverter}s from + * {@link CompositeMessageConverter}. + * + * The primary motivation for this class is to provide an avro based {@link Serde} that is + * compatible with the schema registry that Spring Cloud Stream provides. When using the + * schema registry support from Spring Cloud Stream in a Kafka Streams binder based + * application, the applications can deserialize the incoming Kafka Streams records using + * the built in Avro {@link MessageConverter}. However, this same message conversion + * approach will not work downstream in other operations in the topology for Kafka Streams + * as some of them needs a {@link Serde} instance that can talk to the Spring Cloud Stream + * provided Schema Registry. This implementation will solve that problem. + * + * Only Avro and JSON based converters are exposed as binder provided {@link Serde} + * implementations currently. + * + * Users of this class must call the + * {@link MessageConverterDelegateSerde#configure(Map, boolean)} method to configure the + * {@link Serde} object. At the very least the configuration map must include a key called + * "valueClass" to indicate the type of the target object for deserialization. If any + * other content type other than JSON is needed (only Avro is available now other than + * JSON), that needs to be included in the configuration map with the key "contentType". + * For example, + * + *
+ * Map<String, Object> config = new HashMap<>();
+ * config.put("valueClass", Foo.class);
+ * config.put("contentType", "application/avro");
+ * 
+ * + * Then use the above map when calling the configure method. + * + * This class is only intended to be used when writing a Spring Cloud Stream Kafka Streams + * application that uses Spring Cloud Stream schema registry for schema evolution. + * + * An instance of this class is provided as a bean by the binder configuration and + * typically the applications can autowire that bean. This is the expected usage pattern + * of this class. + * + * @param type of the object to marshall + * @author Soby Chacko + * @since 3.0 + * @deprecated in favor of other schema registry providers instead of Spring Cloud Schema Registry. See its motivation above. + */ +@Deprecated +public class MessageConverterDelegateSerde implements Serde { + + private static final String VALUE_CLASS_HEADER = "valueClass"; + + private static final String AVRO_FORMAT = "avro"; + + private static final MimeType DEFAULT_AVRO_MIME_TYPE = new MimeType("application", + "*+" + AVRO_FORMAT); + + private final MessageConverterDelegateDeserializer messageConverterDelegateDeserializer; + + private final MessageConverterDelegateSerializer messageConverterDelegateSerializer; + + public MessageConverterDelegateSerde( + CompositeMessageConverter compositeMessageConverter) { + this.messageConverterDelegateDeserializer = new MessageConverterDelegateDeserializer<>( + compositeMessageConverter); + this.messageConverterDelegateSerializer = new MessageConverterDelegateSerializer<>( + compositeMessageConverter); + } + + @Override + public void configure(Map configs, boolean isKey) { + this.messageConverterDelegateDeserializer.configure(configs, isKey); + this.messageConverterDelegateSerializer.configure(configs, isKey); + } + + @Override + public void close() { + // No-op + } + + @Override + public Serializer serializer() { + return this.messageConverterDelegateSerializer; + } + + @Override + public Deserializer deserializer() { + return this.messageConverterDelegateDeserializer; + } + + private static MimeType resolveMimeType(Map configs) { + if (configs.containsKey(MessageHeaders.CONTENT_TYPE)) { + String contentType = (String) configs.get(MessageHeaders.CONTENT_TYPE); + if (DEFAULT_AVRO_MIME_TYPE.equals(MimeTypeUtils.parseMimeType(contentType))) { + return DEFAULT_AVRO_MIME_TYPE; + } + else if (contentType.contains("avro")) { + return MimeTypeUtils.parseMimeType("application/avro"); + } + else { + return new MimeType("application", "json", StandardCharsets.UTF_8); + } + } + else { + return new MimeType("application", "json", StandardCharsets.UTF_8); + } + } + + /** + * Custom {@link Deserializer} that uses the {@link org.springframework.cloud.stream.converter.CompositeMessageConverterFactory}. + * + * @param parameterized target type for deserialization + */ + private static class MessageConverterDelegateDeserializer implements Deserializer { + + private final MessageConverter messageConverter; + + private MimeType mimeType; + + private Class valueClass; + + MessageConverterDelegateDeserializer( + CompositeMessageConverter compositeMessageConverter) { + this.messageConverter = compositeMessageConverter; + } + + @Override + public void configure(Map configs, boolean isKey) { + Assert.isTrue(configs.containsKey(VALUE_CLASS_HEADER), + "Deserializers must provide a configuration for valueClass."); + final Object valueClass = configs.get(VALUE_CLASS_HEADER); + Assert.isTrue(valueClass instanceof Class, + "Deserializers must provide a valid value for valueClass."); + this.valueClass = (Class) valueClass; + this.mimeType = resolveMimeType(configs); + } + + @SuppressWarnings("unchecked") + @Override + public U deserialize(String topic, byte[] data) { + Message message = MessageBuilder.withPayload(data) + .setHeader(MessageHeaders.CONTENT_TYPE, this.mimeType.toString()) + .build(); + U messageConverted = (U) this.messageConverter.fromMessage(message, + this.valueClass); + Assert.notNull(messageConverted, "Deserialization failed."); + return messageConverted; + } + + @Override + public void close() { + // No-op + } + + } + + /** + * Custom {@link Serializer} that uses the {@link org.springframework.cloud.stream.converter.CompositeMessageConverterFactory}. + * + * @param parameterized type for serialization + */ + private static class MessageConverterDelegateSerializer implements Serializer { + + private final MessageConverter messageConverter; + + private MimeType mimeType; + + MessageConverterDelegateSerializer( + CompositeMessageConverter compositeMessageConverter) { + this.messageConverter = compositeMessageConverter; + } + + @Override + public void configure(Map configs, boolean isKey) { + this.mimeType = resolveMimeType(configs); + } + + @Override + public byte[] serialize(String topic, V data) { + Message message = MessageBuilder.withPayload(data).build(); + Map headers = new HashMap<>(message.getHeaders()); + headers.put(MessageHeaders.CONTENT_TYPE, this.mimeType.toString()); + MessageHeaders messageHeaders = new MessageHeaders(headers); + final Object payload = this.messageConverter + .toMessage(message.getPayload(), messageHeaders).getPayload(); + return (byte[]) payload; + } + + @Override + public void close() { + // No-op + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/resources/META-INF/spring.binders b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/resources/META-INF/spring.binders new file mode 100644 index 000000000..dde457315 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/resources/META-INF/spring.binders @@ -0,0 +1,8 @@ +kstream:\ +org.springframework.cloud.stream.binder.kafka.streams.KStreamBinderConfiguration +ktable:\ +org.springframework.cloud.stream.binder.kafka.streams.KTableBinderConfiguration +globalktable:\ +org.springframework.cloud.stream.binder.kafka.streams.GlobalKTableBinderConfiguration + + diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/resources/META-INF/spring.factories b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/resources/META-INF/spring.factories new file mode 100644 index 000000000..00fdadf9e --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/main/resources/META-INF/spring.factories @@ -0,0 +1,5 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.springframework.cloud.stream.binder.kafka.streams.ExtendedBindingHandlerMappingsProviderAutoConfiguration,\ + org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsBinderSupportAutoConfiguration,\ + org.springframework.cloud.stream.binder.kafka.streams.function.KafkaStreamsFunctionAutoConfiguration,\ + org.springframework.cloud.stream.binder.kafka.streams.endpoint.KafkaStreamsTopologyEndpointAutoConfiguration diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/ExtendedBindingHandlerMappingsProviderAutoConfigurationTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/ExtendedBindingHandlerMappingsProviderAutoConfigurationTests.java new file mode 100644 index 000000000..43f21e243 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/ExtendedBindingHandlerMappingsProviderAutoConfigurationTests.java @@ -0,0 +1,83 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import org.junit.jupiter.api.Test; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests for {@link ExtendedBindingHandlerMappingsProviderAutoConfiguration}. + */ +class ExtendedBindingHandlerMappingsProviderAutoConfigurationTests { + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withUserConfiguration(KafkaStreamsTestApp.class) + .withPropertyValues( + "spring.cloud.stream.kafka.streams.default.consumer.application-id: testApp123", + "spring.cloud.stream.kafka.streams.default.consumer.consumed-as: default-consumer", + "spring.cloud.stream.kafka.streams.default.consumer.materialized-as: default-materializer", + "spring.cloud.stream.kafka.streams.default.producer.produced-as: default-producer", + "spring.cloud.stream.kafka.streams.default.producer.key-serde: default-foo"); + + @Test + void defaultsUsedWhenNoCustomBindingProperties() { + this.contextRunner.run((context) -> { + assertThat(context) + .hasNotFailed() + .hasSingleBean(KafkaStreamsExtendedBindingProperties.class); + KafkaStreamsExtendedBindingProperties extendedBindingProperties = context.getBean(KafkaStreamsExtendedBindingProperties.class); + assertThat(extendedBindingProperties.getExtendedConsumerProperties("process-in-0")) + .hasFieldOrPropertyWithValue("applicationId", "testApp123") + .hasFieldOrPropertyWithValue("consumedAs", "default-consumer") + .hasFieldOrPropertyWithValue("materializedAs", "default-materializer"); + assertThat(extendedBindingProperties.getExtendedProducerProperties("process-out-0")) + .hasFieldOrPropertyWithValue("producedAs", "default-producer") + .hasFieldOrPropertyWithValue("keySerde", "default-foo"); + }); + } + + @Test + void defaultsRespectedWhenCustomBindingProperties() { + this.contextRunner + .withPropertyValues( + "spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.consumed-as: custom-consumer", + "spring.cloud.stream.kafka.streams.bindings.process-out-0.producer.produced-as: custom-producer") + .run((context) -> { + assertThat(context) + .hasNotFailed() + .hasSingleBean(KafkaStreamsExtendedBindingProperties.class); + KafkaStreamsExtendedBindingProperties extendedBindingProperties = context.getBean(KafkaStreamsExtendedBindingProperties.class); + assertThat(extendedBindingProperties.getExtendedConsumerProperties("process-in-0")) + .hasFieldOrPropertyWithValue("applicationId", "testApp123") + .hasFieldOrPropertyWithValue("consumedAs", "custom-consumer") + .hasFieldOrPropertyWithValue("materializedAs", "default-materializer"); + assertThat(extendedBindingProperties.getExtendedProducerProperties("process-out-0")) + .hasFieldOrPropertyWithValue("producedAs", "custom-producer") + .hasFieldOrPropertyWithValue("keySerde", "default-foo"); + }); + } + + @EnableAutoConfiguration + static class KafkaStreamsTestApp { + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsEventTypeRoutingTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsEventTypeRoutingTests.java new file mode 100644 index 000000000..603d43247 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsEventTypeRoutingTests.java @@ -0,0 +1,265 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.streams.kstream.GlobalKTable; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.serializer.JsonDeserializer; +import org.springframework.kafka.support.serializer.JsonSerializer; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.util.Assert; + +import static org.assertj.core.api.Assertions.assertThat; + +public class KafkaStreamsEventTypeRoutingTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "foo-1", "foo-2"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule.getEmbeddedKafka(); + + private static Consumer consumer; + + private static CountDownLatch LATCH = new CountDownLatch(3); + + @BeforeClass + public static void setUp() { + Map consumerProps = KafkaTestUtils.consumerProps("test-group-1", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put("value.deserializer", JsonDeserializer.class); + consumerProps.put(JsonDeserializer.TRUSTED_PACKAGES, "*"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer, "foo-2"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + } + + //See https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1003 for more context on this test. + @Test + public void testRoutingWorksBasedOnEventTypes() { + SpringApplication app = new SpringApplication(EventTypeRoutingTestConfig.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=process", + "--spring.cloud.stream.bindings.process-in-0.destination=foo-1", + "--spring.cloud.stream.bindings.process-out-0.destination=foo-2", + "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.eventTypes=foo,bar", + "--spring.cloud.stream.kafka.streams.binder.functions.process.applicationId=process-id-foo-0", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + senderProps.put("value.serializer", JsonSerializer.class); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("foo-1"); + Foo foo1 = new Foo(); + foo1.setFoo("foo-1"); + Headers headers = new RecordHeaders(); + headers.add(new RecordHeader("event_type", "foo".getBytes())); + + final ProducerRecord producerRecord1 = new ProducerRecord<>("foo-1", 0, 56, foo1, headers); + template.send(producerRecord1); + + Foo foo2 = new Foo(); + foo2.setFoo("foo-2"); + + final ProducerRecord producerRecord2 = new ProducerRecord<>("foo-1", 0, 57, foo2); + template.send(producerRecord2); + + Foo foo3 = new Foo(); + foo3.setFoo("foo-3"); + + final ProducerRecord producerRecord3 = new ProducerRecord<>("foo-1", 0, 58, foo3, headers); + template.send(producerRecord3); + + Foo foo4 = new Foo(); + foo4.setFoo("foo-4"); + Headers headers1 = new RecordHeaders(); + headers1.add(new RecordHeader("event_type", "bar".getBytes())); + + final ProducerRecord producerRecord4 = new ProducerRecord<>("foo-1", 0, 59, foo4, headers1); + template.send(producerRecord4); + + final ConsumerRecords records = KafkaTestUtils.getRecords(consumer); + + assertThat(records.count()).isEqualTo(3); + + List keys = new ArrayList<>(); + List values = new ArrayList<>(); + + records.forEach(integerFooConsumerRecord -> { + keys.add(integerFooConsumerRecord.key()); + values.add(integerFooConsumerRecord.value()); + }); + + assertThat(keys).containsExactlyInAnyOrder(56, 58, 59); + assertThat(values).containsExactlyInAnyOrder(foo1, foo3, foo4); + } + finally { + pf.destroy(); + } + } + } + + @Test + public void testRoutingWorksBasedOnEventTypesConsumer() throws Exception { + SpringApplication app = new SpringApplication(EventTypeRoutingTestConfig.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=consumer", + "--spring.cloud.stream.bindings.consumer-in-0.destination=foo-consumer-1", + "--spring.cloud.stream.kafka.streams.bindings.consumer-in-0.consumer.eventTypes=foo,bar", + "--spring.cloud.stream.kafka.streams.binder.functions.consumer.applicationId=consumer-id-foo-0", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + senderProps.put("value.serializer", JsonSerializer.class); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("foo-consumer-1"); + Foo foo1 = new Foo(); + foo1.setFoo("foo-1"); + Headers headers = new RecordHeaders(); + headers.add(new RecordHeader("event_type", "foo".getBytes())); + + final ProducerRecord producerRecord1 = new ProducerRecord<>("foo-consumer-1", 0, 56, foo1, headers); + template.send(producerRecord1); + + Foo foo2 = new Foo(); + foo2.setFoo("foo-2"); + + final ProducerRecord producerRecord2 = new ProducerRecord<>("foo-consumer-1", 0, 57, foo2); + template.send(producerRecord2); + + Foo foo3 = new Foo(); + foo3.setFoo("foo-3"); + + final ProducerRecord producerRecord3 = new ProducerRecord<>("foo-consumer-1", 0, 58, foo3, headers); + template.send(producerRecord3); + + Foo foo4 = new Foo(); + foo4.setFoo("foo-4"); + Headers headers1 = new RecordHeaders(); + headers1.add(new RecordHeader("event_type", "bar".getBytes())); + + final ProducerRecord producerRecord4 = new ProducerRecord<>("foo-consumer-1", 0, 59, foo4, headers1); + template.send(producerRecord4); + + Assert.isTrue(LATCH.await(10, TimeUnit.SECONDS), "Foo"); + } + finally { + pf.destroy(); + } + } + } + + @EnableAutoConfiguration + public static class EventTypeRoutingTestConfig { + + @Bean + public Function, KStream> process() { + return input -> input; + } + + @Bean + public java.util.function.Consumer> consumer() { + return ktable -> ktable.toStream().foreach((key, value) -> { + LATCH.countDown(); + }); + } + + @Bean + public java.util.function.Consumer> global() { + return ktable -> { + }; + } + + } + + static class Foo { + String foo; + + public String getFoo() { + return foo; + } + + public void setFoo(String foo) { + this.foo = foo; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Foo foo1 = (Foo) o; + return Objects.equals(foo, foo1.foo); + } + + @Override + public int hashCode() { + return Objects.hash(foo); + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionCompositionTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionCompositionTests.java new file mode 100644 index 000000000..f3716d414 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsFunctionCompositionTests.java @@ -0,0 +1,468 @@ +/* + * Copyright 2021-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import java.util.function.Function; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.streams.kstream.ForeachAction; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.util.Assert; + +import static org.assertj.core.api.Assertions.assertThat; + +public class KafkaStreamsFunctionCompositionTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "fooFuncanotherFooFunc-out-0", "bar"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule.getEmbeddedKafka(); + + private static Consumer consumer; + + private static final CountDownLatch countDownLatch1 = new CountDownLatch(1); + private static final CountDownLatch countDownLatch2 = new CountDownLatch(1); + private static final CountDownLatch countDownLatch3 = new CountDownLatch(2); + + @BeforeClass + public static void setUp() { + Map consumerProps = KafkaTestUtils.consumerProps("fn-composition-group", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer, "fooFuncanotherFooFunc-out-0", "bar"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + } + + @Test + public void testBasicFunctionCompositionWithDefaultDestination() throws InterruptedException { + SpringApplication app = new SpringApplication(FunctionCompositionConfig1.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=fooFunc|anotherFooFunc;anotherProcess", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("fooFuncanotherFooFunc-in-0"); + template.sendDefault("foobar!!"); + //Verify non-composed funcions can be run standalone with composed function chains, i.e foo|bar;buzz + template.setDefaultTopic("anotherProcess-in-0"); + template.sendDefault("this is crazy!!!"); + Thread.sleep(1000); + + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, "fooFuncanotherFooFunc-out-0"); + assertThat(cr.value().contains("foobar!!")).isTrue(); + + Assert.isTrue(countDownLatch1.await(5, TimeUnit.SECONDS), "anotherProcess consumer didn't trigger."); + } + finally { + pf.destroy(); + } + } + } + + @Test + public void testBasicFunctionCompositionWithDestinaion() throws InterruptedException { + SpringApplication app = new SpringApplication(FunctionCompositionConfig1.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=fooFunc|anotherFooFunc;anotherProcess", + "--spring.cloud.stream.bindings.fooFuncanotherFooFunc-in-0.destination=foo", + "--spring.cloud.stream.bindings.fooFuncanotherFooFunc-out-0.destination=bar", + "--spring.cloud.stream.bindings.anotherProcess-in-0.destination=buzz", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("foo"); + template.sendDefault("foobar!!"); + template.setDefaultTopic("buzz"); + template.sendDefault("this is crazy!!!"); + Thread.sleep(1000); + + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, "bar"); + assertThat(cr.value().contains("foobar!!")).isTrue(); + + Assert.isTrue(countDownLatch1.await(5, TimeUnit.SECONDS), "anotherProcess consumer didn't trigger."); + } + finally { + pf.destroy(); + } + } + } + + @Test + public void testFunctionToConsumerComposition() throws InterruptedException { + SpringApplication app = new SpringApplication(FunctionCompositionConfig2.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=fooFunc|anotherProcess", + "--spring.cloud.stream.bindings.fooFuncanotherProcess-in-0.destination=foo", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("foo"); + template.sendDefault("foobar!!"); + + Thread.sleep(1000); + + Assert.isTrue(countDownLatch2.await(5, TimeUnit.SECONDS), "anotherProcess consumer didn't trigger."); + } + finally { + pf.destroy(); + } + } + } + + @Test + public void testBiFunctionToConsumerComposition() throws InterruptedException { + SpringApplication app = new SpringApplication(FunctionCompositionConfig3.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=fooBiFunc|anotherProcess", + "--spring.cloud.stream.bindings.fooBiFuncanotherProcess-in-0.destination=foo", + "--spring.cloud.stream.bindings.fooBiFuncanotherProcess-in-1.destination=foo-1", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("foo"); + template.sendDefault("foobar!!"); + + template.setDefaultTopic("foo-1"); + template.sendDefault("another foobar!!"); + + Thread.sleep(1000); + + Assert.isTrue(countDownLatch3.await(5, TimeUnit.SECONDS), "anotherProcess consumer didn't trigger."); + } + finally { + pf.destroy(); + } + } + } + + @Test + public void testChainedFunctionsAsComposed() throws InterruptedException { + SpringApplication app = new SpringApplication(FunctionCompositionConfig4.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=fooBiFunc|anotherFooFunc|yetAnotherFooFunc|lastFunctionInChain", + "--spring.cloud.stream.function.bindings.fooBiFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-in-0=input1", + "--spring.cloud.stream.function.bindings.fooBiFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-in-1=input2", + "--spring.cloud.stream.function.bindings.fooBiFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-out-0=output", + "--spring.cloud.stream.bindings.input1.destination=my-foo-1", + "--spring.cloud.stream.bindings.input2.destination=my-foo-2", + "--spring.cloud.stream.bindings.output.destination=bar", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + senderProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + + template.setDefaultTopic("my-foo-2"); + template.sendDefault("foo-1", "foo2"); + + template.setDefaultTopic("my-foo-1"); + template.sendDefault("foo-1", "foo1"); + + Thread.sleep(1000); + + final ConsumerRecords records = KafkaTestUtils.getRecords(consumer); + assertThat(records.iterator().hasNext()).isTrue(); + assertThat(records.iterator().next().value().equals("foo1foo2From-anotherFooFuncFrom-yetAnotherFooFuncFrom-lastFunctionInChain")).isTrue(); + } + finally { + pf.destroy(); + } + } + } + + @Test + public void testFirstFunctionCurriedThenComposeWithOtherFunctions() throws InterruptedException { + SpringApplication app = new SpringApplication(FunctionCompositionConfig5.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=curriedFunc|anotherFooFunc|yetAnotherFooFunc|lastFunctionInChain", + "--spring.cloud.stream.function.bindings.curriedFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-in-0=input1", + "--spring.cloud.stream.function.bindings.curriedFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-in-1=input2", + "--spring.cloud.stream.function.bindings.curriedFuncanotherFooFuncyetAnotherFooFunclastFunctionInChain-out-0=output", + "--spring.cloud.stream.bindings.input1.destination=my-foo-1", + "--spring.cloud.stream.bindings.input2.destination=my-foo-2", + "--spring.cloud.stream.bindings.output.destination=bar", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + senderProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + + template.setDefaultTopic("my-foo-2"); + template.sendDefault("foo-1", "foo2"); + + Thread.sleep(1000); + + template.setDefaultTopic("my-foo-1"); + template.sendDefault("foo-1", "foo1"); + + Thread.sleep(1000); + + final ConsumerRecords records = KafkaTestUtils.getRecords(consumer); + assertThat(records.iterator().hasNext()).isTrue(); + assertThat(records.iterator().next().value().equals("foo1foo2From-anotherFooFuncFrom-yetAnotherFooFuncFrom-lastFunctionInChain")).isTrue(); + } + finally { + pf.destroy(); + } + } + } + + @Test + public void testFunctionToConsumerCompositionWithFunctionProducesKTable() throws InterruptedException { + SpringApplication app = new SpringApplication(FunctionCompositionConfig6.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=fooFunc|anotherProcess", + "--spring.cloud.stream.bindings.fooFuncanotherProcess-in-0.destination=foo", + "--spring.cloud.stream.bindings.fooFuncanotherProcess-out-0.destination=bar", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + senderProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("foo"); + template.sendDefault("foo", "foobar!!"); + + Thread.sleep(1000); + + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, "bar"); + assertThat(cr.value().contains("foobar!!")).isTrue(); + } + finally { + pf.destroy(); + } + } + } + + @EnableAutoConfiguration + public static class FunctionCompositionConfig1 { + + @Bean + public Function, KStream> fooFunc() { + return input -> input.peek((s, s2) -> { + System.out.println("hello: " + s2); + }); + } + + @Bean + public Function, KStream> anotherFooFunc() { + return input -> input.peek((s, s2) -> System.out.println("hello Foo: " + s2)); + } + + @Bean + public java.util.function.Consumer> anotherProcess() { + return c -> c.foreach((s, s2) -> { + System.out.println("s2s2s2::" + s2); + countDownLatch1.countDown(); + }); + } + } + + @EnableAutoConfiguration + public static class FunctionCompositionConfig2 { + + @Bean + public Function, KStream> fooFunc() { + return input -> input.peek((s, s2) -> { + System.out.println("hello: " + s2); + }); + } + + @Bean + public java.util.function.Consumer> anotherProcess() { + return c -> c.foreach((s, s2) -> { + System.out.println("s2s2s2::" + s2); + countDownLatch2.countDown(); + }); + } + } + + @EnableAutoConfiguration + public static class FunctionCompositionConfig3 { + + @Bean + public BiFunction, KStream, KStream> fooBiFunc() { + return KStream::merge; + } + + @Bean + public java.util.function.Consumer> anotherProcess() { + return c -> c.foreach((s, s2) -> { + System.out.println("s2s2s2::" + s2); + countDownLatch3.countDown(); + }); + } + } + + @EnableAutoConfiguration + public static class FunctionCompositionConfig4 { + + @Bean + public BiFunction, KTable, KStream> fooBiFunc() { + return (a, b) -> a.join(b, (value1, value2) -> value1 + value2); + } + + @Bean + public Function, KStream> anotherFooFunc() { + return input -> input.mapValues(value -> value + "From-anotherFooFunc"); + } + + @Bean + public Function, KStream> yetAnotherFooFunc() { + return input -> input.mapValues(value -> value + "From-yetAnotherFooFunc"); + } + + @Bean + public Function, KStream> lastFunctionInChain() { + return input -> input.mapValues(value -> value + "From-lastFunctionInChain"); + } + } + + @EnableAutoConfiguration + public static class FunctionCompositionConfig5 { + + @Bean + public Function, Function, KStream>> curriedFunc() { + return a -> b -> + a.join(b, (value1, value2) -> value1 + value2); + } + + @Bean + public Function, KStream> anotherFooFunc() { + return input -> input.mapValues(value -> value + "From-anotherFooFunc"); + } + + @Bean + public Function, KStream> yetAnotherFooFunc() { + return input -> input.mapValues(value -> value + "From-yetAnotherFooFunc"); + } + + @Bean + public Function, KStream> lastFunctionInChain() { + return input -> input.mapValues(value -> value + "From-lastFunctionInChain"); + } + } + + @EnableAutoConfiguration + public static class FunctionCompositionConfig6 { + + @Bean + public Function, KTable> fooFunc() { + return ks -> { + ks.foreach(new ForeachAction() { + @Override + public void apply(String key, String value) { + System.out.println(); + } + }); + return ks.toTable(); + }; + } + + @Bean + public Function, KStream> anotherProcess() { + return KTable::toStream; + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java new file mode 100644 index 000000000..48c7f5f28 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsInteractiveQueryIntegrationTests.java @@ -0,0 +1,301 @@ +/* + * Copyright 2017-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.serialization.IntegerSerializer; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.KeyQueryMetadata; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.StoreQueryParameters; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.state.HostInfo; +import org.apache.kafka.streams.state.QueryableStoreType; +import org.apache.kafka.streams.state.QueryableStoreTypes; +import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; +import org.mockito.Mockito; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.core.CleanupConfig; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.serializer.JsonSerde; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.messaging.handler.annotation.SendTo; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.internal.verification.VerificationModeFactory.times; + +/** + * @author Soby Chacko + * @author Gary Russell + * @author Nico Pommerening + */ +public class KafkaStreamsInteractiveQueryIntegrationTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "counts-id"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + private static Consumer consumer; + + @BeforeClass + public static void setUp() throws Exception { + Map consumerProps = KafkaTestUtils.consumerProps("group-id", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "counts-id"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + } + + @Test + public void testStateStoreRetrievalRetry() { + + StreamsBuilderFactoryBean mock = Mockito.mock(StreamsBuilderFactoryBean.class); + KafkaStreams mockKafkaStreams = Mockito.mock(KafkaStreams.class); + Mockito.when(mock.getKafkaStreams()).thenReturn(mockKafkaStreams); + KafkaStreamsRegistry kafkaStreamsRegistry = new KafkaStreamsRegistry(); + kafkaStreamsRegistry.registerKafkaStreams(mock); + Mockito.when(mock.isRunning()).thenReturn(true); + Properties mockProperties = new Properties(); + mockProperties.put(StreamsConfig.APPLICATION_ID_CONFIG, "fooApp"); + Mockito.when(mock.getStreamsConfiguration()).thenReturn(mockProperties); + KafkaStreamsBinderConfigurationProperties binderConfigurationProperties = + new KafkaStreamsBinderConfigurationProperties(new KafkaProperties()); + binderConfigurationProperties.getStateStoreRetry().setMaxAttempts(3); + InteractiveQueryService interactiveQueryService = new InteractiveQueryService(kafkaStreamsRegistry, + binderConfigurationProperties); + + QueryableStoreType> storeType = QueryableStoreTypes.keyValueStore(); + try { + interactiveQueryService.getQueryableStore("foo", storeType); + } + catch (Exception ignored) { + + } + Mockito.verify(mockKafkaStreams, times(3)) + .store(StoreQueryParameters.fromNameAndType("foo", storeType)); + } + + @Test + public void testStateStoreRetrievalRetryForHostInfoService() { + StreamsBuilderFactoryBean mock = Mockito.mock(StreamsBuilderFactoryBean.class); + KafkaStreams mockKafkaStreams = Mockito.mock(KafkaStreams.class); + Mockito.when(mock.getKafkaStreams()).thenReturn(mockKafkaStreams); + KafkaStreamsRegistry kafkaStreamsRegistry = new KafkaStreamsRegistry(); + kafkaStreamsRegistry.registerKafkaStreams(mock); + Mockito.when(mock.isRunning()).thenReturn(true); + Properties mockProperties = new Properties(); + mockProperties.put(StreamsConfig.APPLICATION_ID_CONFIG, "foobarApp-123"); + Mockito.when(mock.getStreamsConfiguration()).thenReturn(mockProperties); + KafkaStreamsBinderConfigurationProperties binderConfigurationProperties = + new KafkaStreamsBinderConfigurationProperties(new KafkaProperties()); + binderConfigurationProperties.getStateStoreRetry().setMaxAttempts(3); + InteractiveQueryService interactiveQueryService = new InteractiveQueryService(kafkaStreamsRegistry, + binderConfigurationProperties); + + QueryableStoreType> storeType = QueryableStoreTypes.keyValueStore(); + final StringSerializer serializer = new StringSerializer(); + try { + interactiveQueryService.getHostInfo("foo", "fooKey", serializer); + } + catch (Exception ignored) { + + } + Mockito.verify(mockKafkaStreams, times(3)) + .queryMetadataForKey("foo", "fooKey", serializer); + } + + @Test + @Ignore + public void testKstreamBinderWithPojoInputAndStringOuput() throws Exception { + SpringApplication app = new SpringApplication(ProductCountApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.input.destination=foos", + "--spring.cloud.stream.bindings.output.destination=counts-id", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId=ProductCountApplication-abc", + "--spring.cloud.stream.kafka.streams.binder.configuration.application.server" + + "=" + embeddedKafka.getBrokersAsString(), + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString()); + try { + receiveAndValidateFoo(context); + } + finally { + context.close(); + } + } + + private void receiveAndValidateFoo(ConfigurableApplicationContext context) { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("foos"); + template.sendDefault("{\"id\":\"123\"}"); + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, + "counts-id"); + assertThat(cr.value().contains("Count for product with ID 123: 1")).isTrue(); + + ProductCountApplication.Foo foo = context + .getBean(ProductCountApplication.Foo.class); + assertThat(foo.getProductStock(123).equals(1L)); + + // perform assertions on HostInfo related methods in InteractiveQueryService + InteractiveQueryService interactiveQueryService = context + .getBean(InteractiveQueryService.class); + HostInfo currentHostInfo = interactiveQueryService.getCurrentHostInfo(); + + assertThat(currentHostInfo.host() + ":" + currentHostInfo.port()) + .isEqualTo(embeddedKafka.getBrokersAsString()); + + final KeyQueryMetadata keyQueryMetadata = interactiveQueryService.getKeyQueryMetadata("prod-id-count-store", + 123, new IntegerSerializer()); + final HostInfo activeHost = keyQueryMetadata.getActiveHost(); + assertThat(activeHost.host() + ":" + activeHost.port()) + .isEqualTo(embeddedKafka.getBrokersAsString()); + + final KafkaStreams kafkaStreams = interactiveQueryService.getKafkaStreams("prod-id-count-store", + 123, new IntegerSerializer()); + assertThat(kafkaStreams).isNotNull(); + assertThat(interactiveQueryService.getKafkaStreams("non-existent-store", + 123, new IntegerSerializer())).isNull(); + + HostInfo hostInfo = interactiveQueryService.getHostInfo("prod-id-count-store", + 123, new IntegerSerializer()); + assertThat(hostInfo.host() + ":" + hostInfo.port()) + .isEqualTo(embeddedKafka.getBrokersAsString()); + + HostInfo hostInfoFoo = interactiveQueryService + .getHostInfo("prod-id-count-store-foo", 123, new IntegerSerializer()); + assertThat(hostInfoFoo).isNull(); + + final List hostInfos = interactiveQueryService.getAllHostsInfo("prod-id-count-store"); + assertThat(hostInfos.size()).isEqualTo(1); + final HostInfo hostInfo1 = hostInfos.get(0); + assertThat(hostInfo1.host() + ":" + hostInfo1.port()) + .isEqualTo(embeddedKafka.getBrokersAsString()); + + } + + @EnableBinding(KafkaStreamsProcessor.class) + @EnableAutoConfiguration + public static class ProductCountApplication { + + @StreamListener("input") + @SendTo("output") + public KStream process(KStream input) { + + return input.filter((key, product) -> product.getId() == 123) + .map((key, value) -> new KeyValue<>(value.id, value)) + .groupByKey(Grouped.with(new Serdes.IntegerSerde(), + new JsonSerde<>(Product.class))) + .count(Materialized.as("prod-id-count-store")).toStream() + .map((key, value) -> new KeyValue<>(null, + "Count for product with ID 123: " + value)); + } + + @Bean + public Foo foo(InteractiveQueryService interactiveQueryService) { + return new Foo(interactiveQueryService); + } + + @Bean + public CleanupConfig cleanupConfig() { + return new CleanupConfig(false, true); + } + + static class Foo { + + InteractiveQueryService interactiveQueryService; + + Foo(InteractiveQueryService interactiveQueryService) { + this.interactiveQueryService = interactiveQueryService; + } + + public Long getProductStock(Integer id) { + ReadOnlyKeyValueStore keyValueStore = interactiveQueryService + .getQueryableStore("prod-id-count-store", + QueryableStoreTypes.keyValueStore()); + + return (Long) keyValueStore.get(id); + } + + } + } + + static class Product { + + Integer id; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/MultipleFunctionsInSameAppTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/MultipleFunctionsInSameAppTests.java new file mode 100644 index 000000000..b64d4c406 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/MultipleFunctionsInSameAppTests.java @@ -0,0 +1,248 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams; + +import java.lang.reflect.Field; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.Function; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.KStream; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.core.ResolvableType; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.util.Assert; +import org.springframework.util.ReflectionUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +public class MultipleFunctionsInSameAppTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "coffee", "electronics"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule.getEmbeddedKafka(); + + private static Consumer consumer; + + private static CountDownLatch countDownLatch = new CountDownLatch(2); + + @BeforeClass + public static void setUp() { + Map consumerProps = KafkaTestUtils.consumerProps("purchase-groups", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer, "coffee", "electronics"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + } + + @Test + @SuppressWarnings("unchecked") + public void testMultiFunctionsInSameApp() throws InterruptedException { + SpringApplication app = new SpringApplication(MultipleFunctionsInSameApp.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=process;analyze;anotherProcess;yetAnotherProcess", + "--spring.cloud.stream.bindings.process-in-0.destination=purchases", + "--spring.cloud.stream.bindings.process-out-0.destination=coffee", + "--spring.cloud.stream.bindings.process-out-1.destination=electronics", + "--spring.cloud.stream.bindings.analyze-in-0.destination=coffee", + "--spring.cloud.stream.bindings.analyze-in-1.destination=electronics", + "--spring.cloud.stream.kafka.streams.binder.functions.analyze.applicationId=analyze-id-0", + "--spring.cloud.stream.kafka.streams.binder.functions.process.applicationId=process-id-0", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.bindings.process-in-0.consumer.concurrency=2", + "--spring.cloud.stream.bindings.analyze-in-0.consumer.concurrency=1", + "--spring.cloud.stream.kafka.streams.binder.configuration.num.stream.threads=3", + "--spring.cloud.stream.kafka.streams.binder.functions.process.configuration.client.id=process-client", + "--spring.cloud.stream.kafka.streams.binder.functions.analyze.configuration.client.id=analyze-client", + "--spring.cloud.stream.kafka.streams.binder.functions.anotherProcess.configuration.client.id=anotherProcess-client", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + receiveAndValidate("purchases", "coffee", "electronics"); + + StreamsBuilderFactoryBean processStreamsBuilderFactoryBean = context + .getBean("&stream-builder-process", StreamsBuilderFactoryBean.class); + + StreamsBuilderFactoryBean analyzeStreamsBuilderFactoryBean = context + .getBean("&stream-builder-analyze", StreamsBuilderFactoryBean.class); + + StreamsBuilderFactoryBean anotherProcessStreamsBuilderFactoryBean = context + .getBean("&stream-builder-anotherProcess", StreamsBuilderFactoryBean.class); + + final Properties processStreamsConfiguration = processStreamsBuilderFactoryBean.getStreamsConfiguration(); + final Properties analyzeStreamsConfiguration = analyzeStreamsBuilderFactoryBean.getStreamsConfiguration(); + final Properties anotherProcessStreamsConfiguration = anotherProcessStreamsBuilderFactoryBean.getStreamsConfiguration(); + + assertThat(processStreamsConfiguration.getProperty("client.id")).isEqualTo("process-client"); + assertThat(analyzeStreamsConfiguration.getProperty("client.id")).isEqualTo("analyze-client"); + + Integer concurrency = (Integer) processStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG); + assertThat(concurrency).isEqualTo(2); + concurrency = (Integer) analyzeStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG); + assertThat(concurrency).isEqualTo(1); + assertThat(anotherProcessStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG)).isEqualTo("3"); + + final KafkaStreamsBindingInformationCatalogue catalogue = context.getBean(KafkaStreamsBindingInformationCatalogue.class); + Field field = ReflectionUtils.findField(KafkaStreamsBindingInformationCatalogue.class, "outboundKStreamResolvables", Map.class); + ReflectionUtils.makeAccessible(field); + final Map outboundKStreamResolvables = (Map) ReflectionUtils.getField(field, catalogue); + // Since we have 2 functions with return types -- one is an array return type with 2 bindings -- assert that + // the catalogue contains outbound type information for all the 3 different bindings. + assertThat(outboundKStreamResolvables.size()).isEqualTo(3); + } + } + + @Test + public void testMultiFunctionsInSameAppWithMultiBinders() throws Exception { + SpringApplication app = new SpringApplication(MultipleFunctionsInSameApp.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=process;analyze", + "--spring.cloud.stream.bindings.process-in-0.destination=purchases", + "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.startOffset=latest", + "--spring.cloud.stream.bindings.process-in-0.binder=kafka1", + "--spring.cloud.stream.bindings.process-out-0.destination=coffee", + "--spring.cloud.stream.bindings.process-out-0.binder=kafka1", + "--spring.cloud.stream.bindings.process-out-1.destination=electronics", + "--spring.cloud.stream.bindings.process-out-1.binder=kafka1", + "--spring.cloud.stream.bindings.analyze-in-0.destination=coffee", + "--spring.cloud.stream.bindings.analyze-in-0.binder=kafka2", + "--spring.cloud.stream.bindings.analyze-in-1.destination=electronics", + "--spring.cloud.stream.bindings.analyze-in-1.binder=kafka2", + "--spring.cloud.stream.bindings.analyze-in-0.consumer.concurrency=2", + "--spring.cloud.stream.binders.kafka1.type=kstream", + "--spring.cloud.stream.binders.kafka1.environment.spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString(), + "--spring.cloud.stream.binders.kafka1.environment.spring.cloud.stream.kafka.streams.binder.applicationId=my-app-1", + "--spring.cloud.stream.binders.kafka1.environment.spring.cloud.stream.kafka.streams.binder.configuration.client.id=process-client", + "--spring.cloud.stream.binders.kafka2.type=kstream", + "--spring.cloud.stream.binders.kafka2.environment.spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString(), + "--spring.cloud.stream.binders.kafka2.environment.spring.cloud.stream.kafka.streams.binder.applicationId=my-app-2", + "--spring.cloud.stream.binders.kafka2.environment.spring.cloud.stream.kafka.streams.binder.configuration.client.id=analyze-client")) { + + Thread.sleep(1000); + receiveAndValidate("purchases", "coffee", "electronics"); + + StreamsBuilderFactoryBean processStreamsBuilderFactoryBean = context + .getBean("&stream-builder-process", StreamsBuilderFactoryBean.class); + + StreamsBuilderFactoryBean analyzeStreamsBuilderFactoryBean = context + .getBean("&stream-builder-analyze", StreamsBuilderFactoryBean.class); + + final Properties processStreamsConfiguration = processStreamsBuilderFactoryBean.getStreamsConfiguration(); + final Properties analyzeStreamsConfiguration = analyzeStreamsBuilderFactoryBean.getStreamsConfiguration(); + + assertThat(processStreamsConfiguration.getProperty("application.id")).isEqualTo("my-app-1"); + assertThat(analyzeStreamsConfiguration.getProperty("application.id")).isEqualTo("my-app-2"); + assertThat(processStreamsConfiguration.getProperty("client.id")).isEqualTo("process-client"); + assertThat(analyzeStreamsConfiguration.getProperty("client.id")).isEqualTo("analyze-client"); + + Integer concurrency = (Integer) analyzeStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG); + assertThat(concurrency).isEqualTo(2); + + concurrency = (Integer) processStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG); + assertThat(concurrency).isNull(); //thus default to 1 by Kafka Streams. + } + } + + private void receiveAndValidate(String in, String... out) throws InterruptedException { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic(in); + template.sendDefault("coffee"); + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, out[0]); + assertThat(cr.value().contains("coffee")).isTrue(); + + template.sendDefault("electronics"); + cr = KafkaTestUtils.getSingleRecord(consumer, out[1]); + assertThat(cr.value().contains("electronics")).isTrue(); + + Assert.isTrue(countDownLatch.await(5, TimeUnit.SECONDS), "Analyze (BiConsumer) method didn't receive all the expected records"); + } + finally { + pf.destroy(); + } + } + + @EnableAutoConfiguration + public static class MultipleFunctionsInSameApp { + + @Bean + public Function, KStream[]> process() { + return input -> input.branch( + (s, p) -> p.equalsIgnoreCase("coffee"), + (s, p) -> p.equalsIgnoreCase("electronics")); + } + + @Bean + public Function, KStream> yetAnotherProcess() { + return input -> input.map((k, v) -> new KeyValue<>("foo", 1L)); + } + + @Bean + public BiConsumer, KStream> analyze() { + return (coffee, electronics) -> { + coffee.foreach((s, p) -> countDownLatch.countDown()); + electronics.foreach((s, p) -> countDownLatch.countDown()); + }; + } + + @Bean + public java.util.function.Consumer> anotherProcess() { + return c -> { + + }; + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderBootstrapTest.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderBootstrapTest.java new file mode 100644 index 000000000..7d851cff7 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderBootstrapTest.java @@ -0,0 +1,190 @@ +/* + * Copyright 2018-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.bootstrap; + +import java.util.Map; +import java.util.Properties; +import java.util.function.Consumer; + +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.type.TypeFactory; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.security.JaasUtils; +import org.apache.kafka.streams.kstream.GlobalKTable; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.stream.binder.kafka.streams.KeyValueSerdeResolver; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.support.mapping.DefaultJackson2JavaTypeMapper; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +/** + * @author Soby Chacko + * @author Eduard Domínguez + */ +public class KafkaStreamsBinderBootstrapTest { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, 10); + + @Before + public void before() { + System.clearProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM); + } + + @Test + public void testKStreamBinderWithCustomEnvironmentCanStart() { + ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder( + SimpleKafkaStreamsApplication.class).web(WebApplicationType.NONE).run( + "--spring.cloud.function.definition=input1;input2;input3", + "--spring.cloud.stream.kafka.streams.bindings.input1-in-0.consumer.application-id" + + "=testKStreamBinderWithCustomEnvironmentCanStart", + "--spring.cloud.stream.kafka.streams.bindings.input2-in-0.consumer.application-id" + + "=testKStreamBinderWithCustomEnvironmentCanStart-foo", + "--spring.cloud.stream.kafka.streams.bindings.input3-in-0.consumer.application-id" + + "=testKStreamBinderWithCustomEnvironmentCanStart-foobar", + "--spring.cloud.stream.bindings.input1-in-0.destination=foo", + "--spring.cloud.stream.bindings.input1-in-0.binder=kstreamBinder", + "--spring.cloud.stream.binders.kstreamBinder.type=kstream", + "--spring.cloud.stream.binders.kstreamBinder.environment" + + ".spring.cloud.stream.kafka.streams.binder.brokers" + + "=" + embeddedKafka.getEmbeddedKafka().getBrokersAsString(), + "--spring.cloud.stream.bindings.input2-in-0.destination=bar", + "--spring.cloud.stream.bindings.input2-in-0.binder=ktableBinder", + "--spring.cloud.stream.binders.ktableBinder.type=ktable", + "--spring.cloud.stream.binders.ktableBinder.environment" + + ".spring.cloud.stream.kafka.streams.binder.brokers" + + "=" + embeddedKafka.getEmbeddedKafka().getBrokersAsString(), + "--spring.cloud.stream.bindings.input3-in-0.destination=foobar", + "--spring.cloud.stream.bindings.input3-in-0.binder=globalktableBinder", + "--spring.cloud.stream.binders.globalktableBinder.type=globalktable", + "--spring.cloud.stream.binders.globalktableBinder.environment" + + ".spring.cloud.stream.kafka.streams.binder.brokers" + + "=" + embeddedKafka.getEmbeddedKafka().getBrokersAsString()); + + applicationContext.close(); + } + + @Test + public void testKafkaStreamsBinderWithStandardConfigurationCanStart() { + ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder( + SimpleKafkaStreamsApplication.class).web(WebApplicationType.NONE).run( + "--spring.cloud.function.definition=input1;input2;input3", + "--spring.cloud.stream.kafka.streams.bindings.input1-in-0.consumer.application-id" + + "=testKafkaStreamsBinderWithStandardConfigurationCanStart", + "--spring.cloud.stream.kafka.streams.bindings.input2-in-0.consumer.application-id" + + "=testKafkaStreamsBinderWithStandardConfigurationCanStart-foo", + "--spring.cloud.stream.kafka.streams.bindings.input3-in-0.consumer.application-id" + + "=testKafkaStreamsBinderWithStandardConfigurationCanStart-foobar", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getEmbeddedKafka().getBrokersAsString()); + + applicationContext.close(); + } + + @Test + @SuppressWarnings("unchecked") + public void testStreamConfigGlobalProperties_GH1149() { + ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder( + SimpleKafkaStreamsApplication.class).web(WebApplicationType.NONE).run( + "--spring.cloud.function.definition=input1;input2;input3", + "--spring.cloud.stream.kafka.streams.bindings.input1-in-0.consumer.application-id" + + "=testKafkaStreamsBinderWithStandardConfigurationCanStart", + "--spring.cloud.stream.kafka.streams.bindings.input2-in-0.consumer.application-id" + + "=testKafkaStreamsBinderWithStandardConfigurationCanStart-foo", + "--spring.cloud.stream.kafka.streams.bindings.input2-in-0.consumer.configuration.spring.json.value.type.method=" + this.getClass().getName() + ".determineType", + "--spring.cloud.stream.kafka.streams.bindings.input3-in-0.consumer.application-id" + + "=testKafkaStreamsBinderWithStandardConfigurationCanStart-foobar", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getEmbeddedKafka().getBrokersAsString()); + + Map streamConfigGlobalProperties = applicationContext + .getBean("streamConfigGlobalProperties", Map.class); + // Make sure that global stream configs do not contain individual binding config set on second function. + assertThat(streamConfigGlobalProperties.containsKey("spring.json.value.type.method")).isFalse(); + + // Make sure that only input2 function gets the specific binding property set on it. + final StreamsBuilderFactoryBean input1SBFB = applicationContext.getBean("&stream-builder-input1", StreamsBuilderFactoryBean.class); + final Properties streamsConfiguration1 = input1SBFB.getStreamsConfiguration(); + assertThat(streamsConfiguration1.containsKey("spring.json.value.type.method")).isFalse(); + + final StreamsBuilderFactoryBean input2SBFB = applicationContext.getBean("&stream-builder-input2", StreamsBuilderFactoryBean.class); + final Properties streamsConfiguration2 = input2SBFB.getStreamsConfiguration(); + assertThat(streamsConfiguration2.containsKey("spring.json.value.type.method")).isTrue(); + + final StreamsBuilderFactoryBean input3SBFB = applicationContext.getBean("&stream-builder-input3", StreamsBuilderFactoryBean.class); + final Properties streamsConfiguration3 = input3SBFB.getStreamsConfiguration(); + assertThat(streamsConfiguration3.containsKey("spring.json.value.type.method")).isFalse(); + applicationContext.getBean(KeyValueSerdeResolver.class); + + String configuredSerdeTypeResolver = (String) new DirectFieldAccessor(input2SBFB.getKafkaStreams()) + .getPropertyValue("taskTopology.processorNodes[0].valDeserializer.typeResolver.arg$2"); + + assertThat(this.getClass().getName() + ".determineType").isEqualTo(configuredSerdeTypeResolver); + + String configuredKeyDeserializerFieldName = ((String) new DirectFieldAccessor(input2SBFB.getKafkaStreams()) + .getPropertyValue("taskTopology.processorNodes[0].keyDeserializer.typeMapper.classIdFieldName")); + assertThat(DefaultJackson2JavaTypeMapper.KEY_DEFAULT_CLASSID_FIELD_NAME).isEqualTo(configuredKeyDeserializerFieldName); + + String configuredValueDeserializerFieldName = ((String) new DirectFieldAccessor(input2SBFB.getKafkaStreams()) + .getPropertyValue("taskTopology.processorNodes[0].valDeserializer.typeMapper.classIdFieldName")); + assertThat(DefaultJackson2JavaTypeMapper.DEFAULT_CLASSID_FIELD_NAME).isEqualTo(configuredValueDeserializerFieldName); + + applicationContext.close(); + } + + public static JavaType determineType(byte[] data, Headers headers) { + return TypeFactory.defaultInstance().constructParametricType(Map.class, String.class, String.class); + } + + @SpringBootApplication + static class SimpleKafkaStreamsApplication { + + @Bean + public Consumer> input1() { + return s -> { + // No-op consumer + }; + } + + @Bean + public Consumer, Map>> input2() { + return s -> { + // No-op consumer + }; + } + + @Bean + public Consumer> input3() { + return s -> { + // No-op consumer + }; + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderJaasInitTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderJaasInitTests.java new file mode 100644 index 000000000..7f52699a8 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderJaasInitTests.java @@ -0,0 +1,93 @@ +/* + * Copyright 2021-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.bootstrap; + +import java.util.function.Consumer; + +import javax.security.auth.login.AppConfigurationEntry; + +import org.apache.kafka.common.security.JaasUtils; +import org.apache.kafka.streams.kstream.KStream; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; + +import static org.assertj.core.api.Assertions.assertThat; + +public class KafkaStreamsBinderJaasInitTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, 10); + + private static String JAVA_LOGIN_CONFIG_PARAM_VALUE; + + @BeforeClass + public static void beforeAll() { + JAVA_LOGIN_CONFIG_PARAM_VALUE = System.getProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM); + System.clearProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM); + } + + @AfterClass + public static void afterAll() { + if (JAVA_LOGIN_CONFIG_PARAM_VALUE != null) { + System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, JAVA_LOGIN_CONFIG_PARAM_VALUE); + } + } + + @Test + public void testKafkaStreamsBinderJaasInitialization() { + ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder( + KafkaStreamsBinderJaasInitTestsApplication.class).web(WebApplicationType.NONE).run( + "--spring.cloud.function.definition=foo", + "--spring.cloud.stream.kafka.streams.bindings.foo-in-0.consumer.application-id" + + "=testKafkaStreamsBinderJaasInitialization-jaas-id", + "--spring.cloud.stream.kafka.streams.binder.jaas.loginModule=org.apache.kafka.common.security.plain.PlainLoginModule", + "--spring.cloud.stream.kafka.streams.binder.jaas.options.username=foo", + "--spring.cloud.stream.kafka.streams.binder.jaas.options.password=bar", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getEmbeddedKafka().getBrokersAsString()); + javax.security.auth.login.Configuration configuration = javax.security.auth.login.Configuration + .getConfiguration(); + final AppConfigurationEntry[] kafkaConfiguration = configuration + .getAppConfigurationEntry("KafkaClient"); + assertThat(kafkaConfiguration).hasSize(1); + assertThat(kafkaConfiguration[0].getOptions().get("username")).isEqualTo("foo"); + assertThat(kafkaConfiguration[0].getOptions().get("password")).isEqualTo("bar"); + assertThat(kafkaConfiguration[0].getControlFlag()) + .isEqualTo(AppConfigurationEntry.LoginModuleControlFlag.REQUIRED); + applicationContext.close(); + } + + @SpringBootApplication + static class KafkaStreamsBinderJaasInitTestsApplication { + + @Bean + public Consumer> foo() { + return s -> { + // No-op consumer + }; + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsBinderWordCountBranchesFunctionTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsBinderWordCountBranchesFunctionTests.java new file mode 100644 index 000000000..1f1d443ef --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsBinderWordCountBranchesFunctionTests.java @@ -0,0 +1,213 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.function; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Date; +import java.util.Map; +import java.util.function.Function; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Soby Chacko + */ +public class KafkaStreamsBinderWordCountBranchesFunctionTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "counts", "foo", "bar"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule.getEmbeddedKafka(); + + private static Consumer consumer; + + @BeforeClass + public static void setUp() throws Exception { + Map consumerProps = KafkaTestUtils.consumerProps("groupx", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer, "counts", "foo", "bar"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + } + + @Test + public void testKstreamWordCountWithStringInputAndPojoOuput() throws Exception { + SpringApplication app = new SpringApplication(WordCountProcessorApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.cloud.stream.function.bindings.process-in-0=input", + "--spring.cloud.stream.bindings.input.destination=words", + "--spring.cloud.stream.function.bindings.process-out-0=output1", + "--spring.cloud.stream.bindings.output1.destination=counts", + "--spring.cloud.stream.function.bindings.process-out-1=output2", + "--spring.cloud.stream.bindings.output2.destination=foo", + "--spring.cloud.stream.function.bindings.process-out-2=output3", + "--spring.cloud.stream.bindings.output3.destination=bar", + + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.applicationId" + + "=KafkaStreamsBinderWordCountBranchesFunctionTests-abc", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString()); + try { + receiveAndValidate(context); + } + finally { + context.close(); + } + } + + private void receiveAndValidate(ConfigurableApplicationContext context) throws Exception { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("words"); + template.sendDefault("english"); + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, "counts"); + assertThat(cr.value().contains("\"word\":\"english\",\"count\":1")).isTrue(); + + template.sendDefault("french"); + template.sendDefault("french"); + cr = KafkaTestUtils.getSingleRecord(consumer, "foo"); + assertThat(cr.value().contains("\"word\":\"french\",\"count\":2")).isTrue(); + + template.sendDefault("spanish"); + template.sendDefault("spanish"); + template.sendDefault("spanish"); + cr = KafkaTestUtils.getSingleRecord(consumer, "bar"); + assertThat(cr.value().contains("\"word\":\"spanish\",\"count\":3")).isTrue(); + } + + static class WordCount { + + private String word; + + private long count; + + private Date start; + + private Date end; + + WordCount(String word, long count, Date start, Date end) { + this.word = word; + this.count = count; + this.start = start; + this.end = end; + } + + public String getWord() { + return word; + } + + public void setWord(String word) { + this.word = word; + } + + public long getCount() { + return count; + } + + public void setCount(long count) { + this.count = count; + } + + public Date getStart() { + return start; + } + + public void setStart(Date start) { + this.start = start; + } + + public Date getEnd() { + return end; + } + + public void setEnd(Date end) { + this.end = end; + } + } + + @EnableAutoConfiguration + public static class WordCountProcessorApplication { + + @Bean + @SuppressWarnings({"unchecked"}) + public Function, KStream[]> process() { + + Predicate isEnglish = (k, v) -> v.word.equals("english"); + Predicate isFrench = (k, v) -> v.word.equals("french"); + Predicate isSpanish = (k, v) -> v.word.equals("spanish"); + + return input -> { + final Map> stringKStreamMap = input + .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .groupBy((key, value) -> value) + .windowedBy(TimeWindows.of(Duration.ofSeconds(5))) + .count(Materialized.as("WordCounts-branch")) + .toStream() + .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, + new Date(key.window().start()), new Date(key.window().end())))) + .split() + .branch(isEnglish) + .branch(isFrench) + .branch(isSpanish) + .noDefaultBranch(); + + return stringKStreamMap.values().toArray(new KStream[0]); + }; + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsBinderWordCountFunctionTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsBinderWordCountFunctionTests.java new file mode 100644 index 000000000..65276e189 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsBinderWordCountFunctionTests.java @@ -0,0 +1,448 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.function; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import io.micrometer.core.instrument.MeterRegistry; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.apache.kafka.streams.processor.StreamPartitioner; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.stream.binder.Binding; +import org.springframework.cloud.stream.binder.DefaultBinding; +import org.springframework.cloud.stream.binder.kafka.streams.InteractiveQueryService; +import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsRegistry; +import org.springframework.cloud.stream.binder.kafka.streams.StreamsBuilderFactoryManager; +import org.springframework.cloud.stream.binder.kafka.streams.endpoint.KafkaStreamsTopologyEndpoint; +import org.springframework.cloud.stream.binding.InputBindingLifecycle; +import org.springframework.cloud.stream.binding.OutputBindingLifecycle; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.Lifecycle; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.config.StreamsBuilderFactoryBeanConfigurer; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.util.Assert; + +import static org.assertj.core.api.Assertions.assertThat; + +public class KafkaStreamsBinderWordCountFunctionTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "counts", "counts-1", "counts-2", "counts-5", "counts-6"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule.getEmbeddedKafka(); + + private static Consumer consumer; + + private final static CountDownLatch LATCH = new CountDownLatch(1); + + @BeforeClass + public static void setUp() { + Map consumerProps = KafkaTestUtils.consumerProps("group", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer, "counts", "counts-1", "counts-2", "counts-5", "counts-6"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + } + + @Test + @SuppressWarnings("unchecked") + public void testBasicKStreamTopologyExecution() throws Exception { + SpringApplication app = new SpringApplication(WordCountProcessorApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.process-in-0.destination=words", + "--spring.cloud.stream.bindings.process-out-0.destination=counts", + "--spring.cloud.stream.kafka.streams.binder.application-id=testKstreamWordCountFunction", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.consumerProperties.request.timeout.ms=29000", //for testing ...binder.consumerProperties + "--spring.cloud.stream.kafka.streams.binder.consumerProperties.consumer.value.deserializer=org.apache.kafka.common.serialization.StringDeserializer", + "--spring.cloud.stream.kafka.streams.binder.producerProperties.max.block.ms=90000", //for testing ...binder.producerProperties + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.consumedAs=custom-consumer", + "--spring.cloud.stream.kafka.streams.bindings.process-out-0.producer.producedAs=custom-producer", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + receiveAndValidate("words", "counts"); + + final MeterRegistry meterRegistry = context.getBean(MeterRegistry.class); + Thread.sleep(100); + + assertThat(meterRegistry.getMeters().stream().anyMatch(m -> m.getId().getName().equals("kafka.stream.thread.poll.records.max"))).isTrue(); + assertThat(meterRegistry.getMeters().stream().anyMatch(m -> m.getId().getName().equals("kafka.consumer.network.io.total"))).isTrue(); + assertThat(meterRegistry.getMeters().stream().anyMatch(m -> m.getId().getName().equals("kafka.producer.record.send.total"))).isTrue(); + assertThat(meterRegistry.getMeters().stream().anyMatch(m -> m.getId().getName().equals("kafka.admin.client.network.io.total"))).isTrue(); + + Assert.isTrue(LATCH.await(5, TimeUnit.SECONDS), "Failed to call customizers"); + //Testing topology endpoint + final KafkaStreamsRegistry kafkaStreamsRegistry = context.getBean(KafkaStreamsRegistry.class); + final KafkaStreamsTopologyEndpoint kafkaStreamsTopologyEndpoint = new KafkaStreamsTopologyEndpoint(kafkaStreamsRegistry); + final List topologies = kafkaStreamsTopologyEndpoint.kafkaStreamsTopologies(); + final String topology1 = topologies.get(0); + final String topology2 = kafkaStreamsTopologyEndpoint.kafkaStreamsTopology("testKstreamWordCountFunction"); + assertThat(topology1).isNotEmpty(); + assertThat(topology1).isEqualTo(topology2); + assertThat(topology1.contains("Source: custom-consumer")).isTrue(); + assertThat(topology1.contains("Sink: custom-producer")).isTrue(); + + //verify that ...binder.consumerProperties and ...binder.producerProperties work. + Map streamConfigGlobalProperties = (Map) context.getBean("streamConfigGlobalProperties"); + assertThat(streamConfigGlobalProperties.get("consumer.request.timeout.ms")).isEqualTo("29000"); + assertThat(streamConfigGlobalProperties.get("consumer.value.deserializer")).isEqualTo("org.apache.kafka.common.serialization.StringDeserializer"); + assertThat(streamConfigGlobalProperties.get("producer.max.block.ms")).isEqualTo("90000"); + + InputBindingLifecycle inputBindingLifecycle = context.getBean(InputBindingLifecycle.class); + final Collection> inputBindings = (Collection>) new DirectFieldAccessor(inputBindingLifecycle) + .getPropertyValue("inputBindings"); + assertThat(inputBindings).isNotNull(); + final Optional> theOnlyInputBinding = inputBindings.stream().findFirst(); + assertThat(theOnlyInputBinding.isPresent()).isTrue(); + final DefaultBinding objectBinding = (DefaultBinding) theOnlyInputBinding.get(); + assertThat(objectBinding.getBindingName()).isEqualTo("process-in-0"); + + final Lifecycle lifecycle = (Lifecycle) new DirectFieldAccessor(objectBinding).getPropertyValue("lifecycle"); + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = context.getBean(StreamsBuilderFactoryBean.class); + assertThat(lifecycle).isEqualTo(streamsBuilderFactoryBean); + + OutputBindingLifecycle outputBindingLifecycle = context.getBean(OutputBindingLifecycle.class); + final Collection> outputBindings = (Collection>) new DirectFieldAccessor(outputBindingLifecycle) + .getPropertyValue("outputBindings"); + assertThat(outputBindings).isNotNull(); + final Optional> theOnlyOutputBinding = outputBindings.stream().findFirst(); + assertThat(theOnlyOutputBinding.isPresent()).isTrue(); + final DefaultBinding objectBinding1 = (DefaultBinding) theOnlyOutputBinding.get(); + assertThat(objectBinding1.getBindingName()).isEqualTo("process-out-0"); + + final Lifecycle lifecycle1 = (Lifecycle) new DirectFieldAccessor(objectBinding1).getPropertyValue("lifecycle"); + assertThat(lifecycle1).isEqualTo(streamsBuilderFactoryBean); + } + } + + @Test + public void testKstreamWordCountWithApplicationIdSpecifiedAtDefaultConsumer() { + SpringApplication app = new SpringApplication(WordCountProcessorApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.process-in-0.destination=words-5", + "--spring.cloud.stream.bindings.process-out-0.destination=counts-5", + "--spring.cloud.stream.kafka.streams.default.consumer.application-id=testKstreamWordCountWithApplicationIdSpecifiedAtDefaultConsumer", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.binder.brokers=" + + embeddedKafka.getBrokersAsString())) { + receiveAndValidate("words-5", "counts-5"); + } + } + + @Test + public void testKstreamWordCountFunctionWithCustomProducerStreamPartitioner() throws Exception { + SpringApplication app = new SpringApplication(WordCountProcessorApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.kafka.streams.binder.application-id=testKstreamWordCountFunctionWithCustomProducerStreamPartitioner", + "--spring.cloud.stream.bindings.process-in-0.destination=words-2", + "--spring.cloud.stream.bindings.process-out-0.destination=counts-2", + "--spring.cloud.stream.bindings.process-out-0.producer.partitionCount=2", + "--spring.cloud.stream.kafka.streams.bindings.process-out-0.producer.streamPartitionerBeanName" + + "=streamPartitioner", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("words-2"); + template.sendDefault("foo"); + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, "counts-2"); + assertThat(cr.value().contains("\"word\":\"foo\",\"count\":1")).isTrue(); + assertThat(cr.partition() == 0) .isTrue(); + template.sendDefault("bar"); + cr = KafkaTestUtils.getSingleRecord(consumer, "counts-2"); + assertThat(cr.value().contains("\"word\":\"bar\",\"count\":1")).isTrue(); + assertThat(cr.partition() == 1) .isTrue(); + } + finally { + pf.destroy(); + } + } + } + + @Test + public void testKstreamBinderAutoStartup() throws Exception { + SpringApplication app = new SpringApplication(WordCountProcessorApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.kafka.streams.auto-startup=false", + "--spring.cloud.stream.bindings.process-in-0.destination=words-3", + "--spring.cloud.stream.bindings.process-out-0.destination=counts-3", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + final StreamsBuilderFactoryManager streamsBuilderFactoryManager = context.getBean(StreamsBuilderFactoryManager.class); + assertThat(streamsBuilderFactoryManager.isAutoStartup()).isFalse(); + assertThat(streamsBuilderFactoryManager.isRunning()).isFalse(); + } + } + + @Test + public void testKstreamIndividualBindingAutoStartup() throws Exception { + SpringApplication app = new SpringApplication(WordCountProcessorApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.process-in-0.destination=words-4", + "--spring.cloud.stream.bindings.process-in-0.consumer.auto-startup=false", + "--spring.cloud.stream.bindings.process-out-0.destination=counts-4", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + final StreamsBuilderFactoryBean streamsBuilderFactoryBean = context.getBean(StreamsBuilderFactoryBean.class); + assertThat(streamsBuilderFactoryBean.isRunning()).isFalse(); + streamsBuilderFactoryBean.start(); + assertThat(streamsBuilderFactoryBean.isRunning()).isTrue(); + } + } + + // The following test verifies the fixes made for this issue: + // https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/774 + @Test + public void testOutboundNullValueIsHandledGracefully() + throws Exception { + SpringApplication app = new SpringApplication(OutboundNullApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.process-in-0.destination=words-6", + "--spring.cloud.stream.bindings.process-out-0.destination=counts-6", + "--spring.cloud.stream.bindings.process-out-0.producer.useNativeEncoding=false", + "--spring.cloud.stream.kafka.streams.default.consumer.application-id=testOutboundNullValueIsHandledGracefully", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.binder.brokers=" + + embeddedKafka.getBrokersAsString())) { + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("words-6"); + template.sendDefault("foobar"); + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, + "counts-6"); + assertThat(cr.value() == null).isTrue(); + } + finally { + pf.destroy(); + } + } + } + + private void receiveAndValidate(String in, String out) { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic(in); + template.sendDefault("foobar"); + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, out); + assertThat(cr.value().contains("\"word\":\"foobar\",\"count\":1")).isTrue(); + } + finally { + pf.destroy(); + } + } + + static class WordCount { + + private String word; + + private long count; + + private Date start; + + private Date end; + + WordCount(String word, long count, Date start, Date end) { + this.word = word; + this.count = count; + this.start = start; + this.end = end; + } + + public String getWord() { + return word; + } + + public void setWord(String word) { + this.word = word; + } + + public long getCount() { + return count; + } + + public void setCount(long count) { + this.count = count; + } + + public Date getStart() { + return start; + } + + public void setStart(Date start) { + this.start = start; + } + + public Date getEnd() { + return end; + } + + public void setEnd(Date end) { + this.end = end; + } + } + + @EnableAutoConfiguration + public static class WordCountProcessorApplication { + + @Autowired + InteractiveQueryService interactiveQueryService; + + @Bean + public Function, KStream> process() { + + return input -> input + .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .map((key, value) -> new KeyValue<>(value, value)) + .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) + .windowedBy(TimeWindows.of(Duration.ofMillis(5000))) + .count(Materialized.as("foo-WordCounts")) + .toStream() + .map((key, value) -> new KeyValue<>(key.key(), new WordCount(key.key(), value, + new Date(key.window().start()), new Date(key.window().end())))); + } + + @Bean + public StreamsBuilderFactoryBeanConfigurer customizer() { + return fb -> { + try { + fb.setStateListener((newState, oldState) -> { + + }); + fb.getObject(); //make sure no exception is thrown at this call. + KafkaStreamsBinderWordCountFunctionTests.LATCH.countDown(); + + } + catch (Exception e) { + //Nothing to do - When the exception is thrown above, the latch won't be count down. + } + }; + } + + @Bean + public StreamPartitioner streamPartitioner() { + return (t, k, v, n) -> k.equals("foo") ? 0 : 1; + } + } + + @EnableAutoConfiguration + static class OutboundNullApplication { + + @Bean + public Function, KStream> process() { + return input -> input + .flatMapValues( + value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .map((key, value) -> new KeyValue<>(value, value)) + .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) + .windowedBy(TimeWindows.of(Duration.ofSeconds(5))).count(Materialized.as("foobar-WordCounts")) + .toStream() + .map((key, value) -> new KeyValue<>(null, null)); + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsComponentBeansTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsComponentBeansTests.java new file mode 100644 index 000000000..cc4a5259d --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsComponentBeansTests.java @@ -0,0 +1,347 @@ +/* + * Copyright 2021-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.function; + +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.KStream; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Soby Chacko + */ +public class KafkaStreamsComponentBeansTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "testFunctionComponent-out", "testBiFunctionComponent-out", "testCurriedFunctionWithFunctionTerminal-out"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule.getEmbeddedKafka(); + + private static Consumer consumer1; + private static Consumer consumer2; + private static Consumer consumer3; + + private final static CountDownLatch LATCH_1 = new CountDownLatch(1); + private final static CountDownLatch LATCH_2 = new CountDownLatch(2); + private final static CountDownLatch LATCH_3 = new CountDownLatch(3); + + @BeforeClass + public static void setUp() { + Map consumerProps = KafkaTestUtils.consumerProps("group", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + consumer1 = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer1, "testFunctionComponent-out"); + + Map consumerProps1 = KafkaTestUtils.consumerProps("group-x", "false", + embeddedKafka); + consumerProps1.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps1.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + DefaultKafkaConsumerFactory cf1 = new DefaultKafkaConsumerFactory<>(consumerProps1); + consumer2 = cf1.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer2, "testBiFunctionComponent-out"); + + Map consumerProps2 = KafkaTestUtils.consumerProps("group-y", "false", + embeddedKafka); + consumerProps2.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps2.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + DefaultKafkaConsumerFactory cf2 = new DefaultKafkaConsumerFactory<>(consumerProps2); + consumer3 = cf2.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer3, "testCurriedFunctionWithFunctionTerminal-out"); + } + + @AfterClass + public static void tearDown() { + consumer1.close(); + consumer2.close(); + consumer3.close(); + } + + @Test + public void testFunctionComponent() { + SpringApplication app = new SpringApplication(FunctionAsComponent.class); + app.setWebApplicationType(WebApplicationType.NONE); + try (ConfigurableApplicationContext ignored = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.foo-in-0.destination=testFunctionComponent-in", + "--spring.cloud.stream.bindings.foo-out-0.destination=testFunctionComponent-out", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("testFunctionComponent-in"); + template.sendDefault("foobar"); + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer1, "testFunctionComponent-out"); + assertThat(cr.value().contains("foobarfoobar")).isTrue(); + } + finally { + pf.destroy(); + } + } + } + + @Test + public void testConsumerComponent() throws Exception { + SpringApplication app = new SpringApplication(ConsumerAsComponent.class); + app.setWebApplicationType(WebApplicationType.NONE); + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.bar-in-0.destination=testConsumerComponent-in", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("testConsumerComponent-in"); + template.sendDefault("foobar"); + Assert.isTrue(LATCH_1.await(10, TimeUnit.SECONDS), "bar"); + } + finally { + pf.destroy(); + } + } + } + + @Test + public void testBiFunctionComponent() { + SpringApplication app = new SpringApplication(BiFunctionAsComponent.class); + app.setWebApplicationType(WebApplicationType.NONE); + try (ConfigurableApplicationContext ignored = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.bazz-in-0.destination=testBiFunctionComponent-in-0", + "--spring.cloud.stream.bindings.bazz-in-1.destination=testBiFunctionComponent-in-1", + "--spring.cloud.stream.bindings.bazz-out-0.destination=testBiFunctionComponent-out", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("testBiFunctionComponent-in-0"); + template.sendDefault("foobar"); + template.setDefaultTopic("testBiFunctionComponent-in-1"); + template.sendDefault("foobar"); + final ConsumerRecords records = KafkaTestUtils.getRecords(consumer2, 10_000, 2); + assertThat(records.count()).isEqualTo(2); + records.forEach(stringStringConsumerRecord -> assertThat(stringStringConsumerRecord.value().contains("foobar")).isTrue()); + } + finally { + pf.destroy(); + } + } + } + + @Test + public void testBiConsumerComponent() throws Exception { + SpringApplication app = new SpringApplication(BiConsumerAsComponent.class); + app.setWebApplicationType(WebApplicationType.NONE); + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.buzz-in-0.destination=testBiConsumerComponent-in-0", + "--spring.cloud.stream.bindings.buzz-in-1.destination=testBiConsumerComponent-in-1", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("testBiConsumerComponent-in-0"); + template.sendDefault("foobar"); + template.setDefaultTopic("testBiConsumerComponent-in-1"); + template.sendDefault("foobar"); + Assert.isTrue(LATCH_2.await(10, TimeUnit.SECONDS), "bar"); + } + finally { + pf.destroy(); + } + } + } + + @Test + public void testCurriedFunctionWithConsumerTerminal() throws Exception { + SpringApplication app = new SpringApplication(CurriedFunctionWithConsumerTerminal.class); + app.setWebApplicationType(WebApplicationType.NONE); + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.curriedConsumer-in-0.destination=testCurriedFunctionWithConsumerTerminal-in-0", + "--spring.cloud.stream.bindings.curriedConsumer-in-1.destination=testCurriedFunctionWithConsumerTerminal-in-1", + "--spring.cloud.stream.bindings.curriedConsumer-in-2.destination=testCurriedFunctionWithConsumerTerminal-in-2", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("testCurriedFunctionWithConsumerTerminal-in-0"); + template.sendDefault("foobar"); + template.setDefaultTopic("testCurriedFunctionWithConsumerTerminal-in-1"); + template.sendDefault("foobar"); + template.setDefaultTopic("testCurriedFunctionWithConsumerTerminal-in-2"); + template.sendDefault("foobar"); + Assert.isTrue(LATCH_3.await(10, TimeUnit.SECONDS), "bar"); + } + finally { + pf.destroy(); + } + } + } + + @Test + public void testCurriedFunctionWithFunctionTerminal() { + SpringApplication app = new SpringApplication(CurriedFunctionWithFunctionTerminal.class); + app.setWebApplicationType(WebApplicationType.NONE); + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.curriedFunction-in-0.destination=testCurriedFunctionWithFunctionTerminal-in-0", + "--spring.cloud.stream.bindings.curriedFunction-in-1.destination=testCurriedFunctionWithFunctionTerminal-in-1", + "--spring.cloud.stream.bindings.curriedFunction-in-2.destination=testCurriedFunctionWithFunctionTerminal-in-2", + "--spring.cloud.stream.bindings.curriedFunction-out-0.destination=testCurriedFunctionWithFunctionTerminal-out", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("testCurriedFunctionWithFunctionTerminal-in-0"); + template.sendDefault("foobar"); + template.setDefaultTopic("testCurriedFunctionWithFunctionTerminal-in-1"); + template.sendDefault("foobar"); + template.setDefaultTopic("testCurriedFunctionWithFunctionTerminal-in-2"); + template.sendDefault("foobar"); + final ConsumerRecords records = KafkaTestUtils.getRecords(consumer3, 10_000, 3); + assertThat(records.count()).isEqualTo(3); + records.forEach(stringStringConsumerRecord -> assertThat(stringStringConsumerRecord.value().contains("foobar")).isTrue()); + } + finally { + pf.destroy(); + } + } + } + + @Component("foo") + @EnableAutoConfiguration + public static class FunctionAsComponent implements Function, + KStream> { + + @Override + public KStream apply(KStream stringIntegerKStream) { + return stringIntegerKStream.map((integer, s) -> new KeyValue<>(s, s + s)); + } + } + + @Component("bar") + @EnableAutoConfiguration + public static class ConsumerAsComponent implements java.util.function.Consumer> { + + @Override + public void accept(KStream integerStringKStream) { + integerStringKStream.foreach((integer, s) -> LATCH_1.countDown()); + } + } + + @Component("bazz") + @EnableAutoConfiguration + public static class BiFunctionAsComponent implements BiFunction, KStream, KStream> { + + @Override + public KStream apply(KStream stringStringKStream, KStream stringStringKStream2) { + return stringStringKStream.merge(stringStringKStream2); + } + } + + @Component("buzz") + @EnableAutoConfiguration + public static class BiConsumerAsComponent implements BiConsumer, KStream> { + + @Override + public void accept(KStream stringStringKStream, KStream stringStringKStream2) { + final KStream merged = stringStringKStream.merge(stringStringKStream2); + merged.foreach((s, s2) -> LATCH_2.countDown()); + } + } + + @Component("curriedConsumer") + @EnableAutoConfiguration + public static class CurriedFunctionWithConsumerTerminal implements Function, + Function, + java.util.function.Consumer>>> { + + @Override + public Function, java.util.function.Consumer>> apply(KStream stringStringKStream) { + return stringStringKStream1 -> stringStringKStream2 -> { + final KStream merge1 = stringStringKStream.merge(stringStringKStream1); + final KStream merged2 = merge1.merge(stringStringKStream2); + merged2.foreach((s1, s2) -> LATCH_3.countDown()); + }; + } + } + + @Component("curriedFunction") + @EnableAutoConfiguration + public static class CurriedFunctionWithFunctionTerminal implements Function, + Function, + java.util.function.Function, KStream>>> { + + @Override + public Function, Function, KStream>> apply(KStream stringStringKStream) { + return stringStringKStream1 -> stringStringKStream2 -> { + final KStream merge1 = stringStringKStream.merge(stringStringKStream1); + return merge1.merge(stringStringKStream2); + }; + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionStateStoreTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionStateStoreTests.java new file mode 100644 index 000000000..36747368a --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionStateStoreTests.java @@ -0,0 +1,187 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.function; + +import java.time.Duration; +import java.util.Map; + +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.processor.Processor; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.ProcessorSupplier; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.StoreBuilder; +import org.apache.kafka.streams.state.Stores; +import org.apache.kafka.streams.state.WindowStore; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +public class KafkaStreamsFunctionStateStoreTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "counts"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule.getEmbeddedKafka(); + + @Test + public void testKafkaStreamsFuncionWithMultipleStateStores() throws Exception { + SpringApplication app = new SpringApplication(StateStoreTestApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=biConsumerBean;hello", + "--spring.cloud.stream.bindings.biConsumerBean-in-0.destination=words", + "--spring.cloud.stream.bindings.hello-in-0.destination=words", + "--spring.cloud.stream.kafka.streams.binder.functions.changed.applicationId=testKafkaStreamsFuncionWithMultipleStateStores-123", + "--spring.cloud.stream.kafka.streams.binder.functions.hello.applicationId=testKafkaStreamsFuncionWithMultipleStateStores-456", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + receiveAndValidate(context); + } + } + + private void receiveAndValidate(ConfigurableApplicationContext context) throws Exception { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("words"); + template.sendDefault(1, "foobar"); + Thread.sleep(2000L); + StateStoreTestApplication processorApplication = context + .getBean(StateStoreTestApplication.class); + + KeyValueStore state1 = processorApplication.state1; + assertThat(processorApplication.processed1).isTrue(); + assertThat(state1 != null).isTrue(); + assertThat(state1.name()).isEqualTo("my-store"); + WindowStore state2 = processorApplication.state2; + assertThat(state2 != null).isTrue(); + assertThat(state2.name()).isEqualTo("other-store"); + assertThat(state2.persistent()).isTrue(); + + KeyValueStore state3 = processorApplication.state1; + assertThat(processorApplication.processed2).isTrue(); + assertThat(state3 != null).isTrue(); + assertThat(state3.name()).isEqualTo("my-store"); + WindowStore state4 = processorApplication.state2; + assertThat(state4 != null).isTrue(); + assertThat(state4.name()).isEqualTo("other-store"); + assertThat(state4.persistent()).isTrue(); + } + finally { + pf.destroy(); + } + } + + @EnableAutoConfiguration + public static class StateStoreTestApplication { + + KeyValueStore state1; + WindowStore state2; + + KeyValueStore state3; + WindowStore state4; + + boolean processed1; + boolean processed2; + + @Bean(name = "biConsumerBean") + public java.util.function.BiConsumer, KStream> process() { + return (input0, input1) -> + input0.process((ProcessorSupplier) () -> new Processor() { + @Override + @SuppressWarnings("unchecked") + public void init(ProcessorContext context) { + state1 = (KeyValueStore) context.getStateStore("my-store"); + state2 = (WindowStore) context.getStateStore("other-store"); + } + + @Override + public void process(Object key, String value) { + processed1 = true; + } + + @Override + public void close() { + + } + }, "my-store", "other-store"); + } + + @Bean + public java.util.function.Consumer> hello() { + return input -> { + input.toStream().process(() -> new Processor() { + @Override + @SuppressWarnings("unchecked") + public void init(ProcessorContext context) { + state3 = (KeyValueStore) context.getStateStore("my-store"); + state4 = (WindowStore) context.getStateStore("other-store"); + } + + @Override + public void process(Object key, String value) { + processed2 = true; + } + + @Override + public void close() { + + } + }, "my-store", "other-store"); + }; + } + + @Bean + public StoreBuilder myStore() { + return Stores.keyValueStoreBuilder( + Stores.persistentKeyValueStore("my-store"), Serdes.Long(), + Serdes.Long()); + } + + @Bean + public StoreBuilder otherStore() { + return Stores.windowStoreBuilder( + Stores.persistentWindowStore("other-store", + Duration.ofSeconds(3), Duration.ofSeconds(3), false), Serdes.Long(), + Serdes.Long()); + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsRetryTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsRetryTests.java new file mode 100644 index 000000000..285822314 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsRetryTests.java @@ -0,0 +1,216 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.function; + +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; + +import org.apache.kafka.streams.kstream.GlobalKTable; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.processor.Processor; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.beans.factory.NoSuchBeanDefinitionException; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.stream.annotation.StreamRetryTemplate; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Lazy; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.retry.RetryPolicy; +import org.springframework.retry.backoff.FixedBackOffPolicy; +import org.springframework.retry.policy.SimpleRetryPolicy; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.util.Assert; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; + +public class KafkaStreamsRetryTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true); + + private static final EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule.getEmbeddedKafka(); + + private final static CountDownLatch LATCH1 = new CountDownLatch(2); + private final static CountDownLatch LATCH2 = new CountDownLatch(4); + + @Test + public void testRetryTemplatePerBindingOnKStream() throws Exception { + SpringApplication app = new SpringApplication(RetryTemplatePerConsumerBindingApp.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.function.definition=process", + "--spring.cloud.stream.bindings.process-in-0.destination=words", + "--spring.cloud.stream.bindings.process-in-0.consumer.max-attempts=2", + "--spring.cloud.stream.kafka.streams.default.consumer.application-id=testRetryTemplatePerBindingOnKStream", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + sendAndValidate(LATCH1); + } + } + + @Test + public void testRetryTemplateOnTableTypes() throws Exception { + SpringApplication app = new SpringApplication(RetryTemplatePerConsumerBindingApp.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.function.definition=tableTypes", + "--spring.cloud.stream.kafka.streams.default.consumer.application-id=testRetryTemplateOnTableTypes", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + + assertThat(context.getBean("tableTypes-in-0-RetryTemplate", RetryTemplate.class)).isNotNull(); + assertThat(context.getBean("tableTypes-in-1-RetryTemplate", RetryTemplate.class)).isNotNull(); + } + } + + @Test + public void testRetryTemplateBeanProvidedByTheApp() throws Exception { + SpringApplication app = new SpringApplication(CustomRetryTemplateApp.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.function.definition=process", + "--spring.cloud.stream.bindings.process-in-0.destination=words", + "--spring.cloud.stream.bindings.process-in-0.consumer.retry-template-name=fooRetryTemplate", + "--spring.cloud.stream.kafka.streams.default.consumer.application-id=testRetryTemplateBeanProvidedByTheApp", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + sendAndValidate(LATCH2); + assertThatThrownBy(() -> context.getBean("process-in-0-RetryTemplate", RetryTemplate.class)).isInstanceOf(NoSuchBeanDefinitionException.class); + } + } + + private void sendAndValidate(CountDownLatch latch) throws InterruptedException { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("words"); + template.sendDefault("foobar"); + Assert.isTrue(latch.await(10, TimeUnit.SECONDS), "Foo"); + } + finally { + pf.destroy(); + } + } + + @EnableAutoConfiguration + public static class RetryTemplatePerConsumerBindingApp { + + @Bean + public java.util.function.Consumer> process(@Lazy @Qualifier("process-in-0-RetryTemplate") RetryTemplate retryTemplate) { + + return input -> input + .process(() -> new Processor() { + @Override + public void init(ProcessorContext processorContext) { + } + + @Override + public void process(Object o, String s) { + retryTemplate.execute(context -> { + LATCH1.countDown(); + throw new RuntimeException(); + }); + } + + @Override + public void close() { + } + }); + } + + @Bean + public BiConsumer, GlobalKTable> tableTypes() { + return (t, g) -> { + }; + } + } + + @EnableAutoConfiguration + public static class CustomRetryTemplateApp { + + @Bean + @StreamRetryTemplate + RetryTemplate fooRetryTemplate() { + RetryTemplate retryTemplate = new RetryTemplate(); + + RetryPolicy retryPolicy = new SimpleRetryPolicy(4); + FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy(); + backOffPolicy.setBackOffPeriod(1); + + retryTemplate.setBackOffPolicy(backOffPolicy); + retryTemplate.setRetryPolicy(retryPolicy); + + return retryTemplate; + } + + @Bean + public java.util.function.Consumer> process() { + + return input -> input + .process(() -> new Processor() { + @Override + public void init(ProcessorContext processorContext) { + } + + @Override + public void process(Object o, String s) { + fooRetryTemplate().execute(context -> { + LATCH2.countDown(); + throw new RuntimeException(); + }); + + } + + @Override + public void close() { + } + }); + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/SerdesProvidedAsBeansTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/SerdesProvidedAsBeansTests.java new file mode 100644 index 000000000..f5893ce84 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/SerdesProvidedAsBeansTests.java @@ -0,0 +1,121 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.function; + +import java.lang.reflect.Method; +import java.util.Date; +import java.util.function.Function; + +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.streams.kstream.KStream; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.stream.binder.ConsumerProperties; +import org.springframework.cloud.stream.binder.ProducerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.KeyValueSerdeResolver; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsProducerProperties; +import org.springframework.cloud.stream.config.BindingServiceProperties; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.core.ResolvableType; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.util.Assert; + +public class SerdesProvidedAsBeansTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule.getEmbeddedKafka(); + + @Test + public void testKstreamWordCountFunction() throws NoSuchMethodException { + SpringApplication app = new SpringApplication(SerdeProvidedAsBeanApp.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.process-in-0.destination=purchases", + "--spring.cloud.stream.bindings.process-out-0.destination=coffee", + "--spring.cloud.stream.kafka.streams.binder.functions.process.applicationId=process-id-0", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + + final Method method = SerdeProvidedAsBeanApp.class.getMethod("process"); + + ResolvableType resolvableType = ResolvableType.forMethodReturnType(method, SerdeProvidedAsBeanApp.class); + + final KeyValueSerdeResolver keyValueSerdeResolver = context.getBean(KeyValueSerdeResolver.class); + final BindingServiceProperties bindingServiceProperties = context.getBean(BindingServiceProperties.class); + final KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties = context.getBean(KafkaStreamsExtendedBindingProperties.class); + + final ConsumerProperties consumerProperties = bindingServiceProperties.getBindingProperties("process-in-0").getConsumer(); + final KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties = kafkaStreamsExtendedBindingProperties.getExtendedConsumerProperties("input"); + kafkaStreamsExtendedBindingProperties.getExtendedConsumerProperties("input"); + final Serde inboundValueSerde = keyValueSerdeResolver.getInboundValueSerde(consumerProperties, kafkaStreamsConsumerProperties, resolvableType.getGeneric(0)); + + Assert.isTrue(inboundValueSerde instanceof FooSerde, "Inbound Value Serde is not matched"); + + final ProducerProperties producerProperties = bindingServiceProperties.getBindingProperties("process-out-0").getProducer(); + final KafkaStreamsProducerProperties kafkaStreamsProducerProperties = kafkaStreamsExtendedBindingProperties.getExtendedProducerProperties("output"); + kafkaStreamsExtendedBindingProperties.getExtendedProducerProperties("output"); + final Serde outboundValueSerde = keyValueSerdeResolver.getOutboundValueSerde(producerProperties, kafkaStreamsProducerProperties, resolvableType.getGeneric(1)); + + Assert.isTrue(outboundValueSerde instanceof FooSerde, "Outbound Value Serde is not matched"); + } + } + + static class FooSerde implements Serde { + @Override + public Serializer serializer() { + return null; + } + + @Override + public Deserializer deserializer() { + return null; + } + } + + @EnableAutoConfiguration + public static class SerdeProvidedAsBeanApp { + + @Bean + public Function, KStream> process() { + return input -> input; + } + + @Bean + public Serde fooSerde() { + return new FooSerde<>(); + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/StreamToGlobalKTableFunctionTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/StreamToGlobalKTableFunctionTests.java new file mode 100644 index 000000000..b7eb33172 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/StreamToGlobalKTableFunctionTests.java @@ -0,0 +1,414 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.function; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.LongDeserializer; +import org.apache.kafka.common.serialization.LongSerializer; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.GlobalKTable; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.processor.TimestampExtractor; +import org.apache.kafka.streams.processor.WallclockTimestampExtractor; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.stream.binder.Binder; +import org.springframework.cloud.stream.binder.BinderFactory; +import org.springframework.cloud.stream.binder.ConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder; +import org.springframework.cloud.stream.binder.ProducerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBindingProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.serializer.JsonDeserializer; +import org.springframework.kafka.support.serializer.JsonSerializer; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +public class StreamToGlobalKTableFunctionTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "enriched-order"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule.getEmbeddedKafka(); + + private static Consumer consumer; + + @Test + public void testStreamToGlobalKTable() throws Exception { + SpringApplication app = new SpringApplication(OrderEnricherApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + try (ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=process", + "--spring.cloud.stream.function.bindings.process-in-0=order", + "--spring.cloud.stream.function.bindings.process-in-1=customer", + "--spring.cloud.stream.function.bindings.process-in-2=product", + "--spring.cloud.stream.function.bindings.process-out-0=enriched-order", + "--spring.cloud.stream.bindings.order.destination=orders", + "--spring.cloud.stream.bindings.customer.destination=customers", + "--spring.cloud.stream.bindings.product.destination=products", + "--spring.cloud.stream.bindings.enriched-order.destination=enriched-order", + + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=10000", + "--spring.cloud.stream.kafka.streams.bindings.order.consumer.applicationId=" + + "StreamToGlobalKTableJoinFunctionTests-abc", + + "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.topic.properties.cleanup.policy=compact", + "--spring.cloud.stream.kafka.streams.bindings.process-in-1.consumer.topic.properties.cleanup.policy=compact", + "--spring.cloud.stream.kafka.streams.bindings.process-in-2.consumer.topic.properties.cleanup.policy=compact", + + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + + // Testing certain ancillary configuration of GlobalKTable around topics creation. + // See this issue: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/687 + + BinderFactory binderFactory = context.getBeanFactory() + .getBean(BinderFactory.class); + + Binder kStreamBinder = binderFactory + .getBinder("kstream", KStream.class); + + KafkaStreamsConsumerProperties input = (KafkaStreamsConsumerProperties) ((ExtendedPropertiesBinder) kStreamBinder) + .getExtendedConsumerProperties("process-in-0"); + String cleanupPolicy = input.getTopic().getProperties().get("cleanup.policy"); + + assertThat(cleanupPolicy).isEqualTo("compact"); + + Binder globalKTableBinder = binderFactory + .getBinder("globalktable", GlobalKTable.class); + + KafkaStreamsConsumerProperties inputX = (KafkaStreamsConsumerProperties) ((ExtendedPropertiesBinder) globalKTableBinder) + .getExtendedConsumerProperties("process-in-1"); + String cleanupPolicyX = inputX.getTopic().getProperties().get("cleanup.policy"); + + assertThat(cleanupPolicyX).isEqualTo("compact"); + + KafkaStreamsConsumerProperties inputY = (KafkaStreamsConsumerProperties) ((ExtendedPropertiesBinder) globalKTableBinder) + .getExtendedConsumerProperties("process-in-2"); + String cleanupPolicyY = inputY.getTopic().getProperties().get("cleanup.policy"); + + assertThat(cleanupPolicyY).isEqualTo("compact"); + + + Map senderPropsCustomer = KafkaTestUtils.producerProps(embeddedKafka); + senderPropsCustomer.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class); + senderPropsCustomer.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + JsonSerializer.class); + + DefaultKafkaProducerFactory pfCustomer = + new DefaultKafkaProducerFactory<>(senderPropsCustomer); + KafkaTemplate template = new KafkaTemplate<>(pfCustomer, true); + template.setDefaultTopic("customers"); + for (long i = 0; i < 5; i++) { + final Customer customer = new Customer(); + customer.setName("customer-" + i); + template.sendDefault(i, customer); + } + + Map senderPropsProduct = KafkaTestUtils.producerProps(embeddedKafka); + senderPropsProduct.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class); + senderPropsProduct.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class); + + DefaultKafkaProducerFactory pfProduct = + new DefaultKafkaProducerFactory<>(senderPropsProduct); + KafkaTemplate productTemplate = new KafkaTemplate<>(pfProduct, true); + productTemplate.setDefaultTopic("products"); + + for (long i = 0; i < 5; i++) { + final Product product = new Product(); + product.setName("product-" + i); + productTemplate.sendDefault(i, product); + } + + Map senderPropsOrder = KafkaTestUtils.producerProps(embeddedKafka); + senderPropsOrder.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class); + senderPropsOrder.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class); + + DefaultKafkaProducerFactory pfOrder = new DefaultKafkaProducerFactory<>(senderPropsOrder); + KafkaTemplate orderTemplate = new KafkaTemplate<>(pfOrder, true); + orderTemplate.setDefaultTopic("orders"); + + for (long i = 0; i < 5; i++) { + final Order order = new Order(); + order.setCustomerId(i); + order.setProductId(i); + orderTemplate.sendDefault(i, order); + } + + Map consumerProps = KafkaTestUtils.consumerProps("group", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, LongDeserializer.class); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + JsonDeserializer.class); + consumerProps.put(JsonDeserializer.VALUE_DEFAULT_TYPE, + "org.springframework.cloud.stream.binder.kafka.streams." + + "function.StreamToGlobalKTableFunctionTests.EnrichedOrder"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + + consumer = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "enriched-order"); + + int count = 0; + long start = System.currentTimeMillis(); + List> enrichedOrders = new ArrayList<>(); + do { + ConsumerRecords records = KafkaTestUtils.getRecords(consumer); + count = count + records.count(); + for (ConsumerRecord record : records) { + enrichedOrders.add(new KeyValue<>(record.key(), record.value())); + } + } while (count < 5 && (System.currentTimeMillis() - start) < 30000); + + assertThat(count == 5).isTrue(); + assertThat(enrichedOrders.size() == 5).isTrue(); + + enrichedOrders.sort(Comparator.comparing(o -> o.key)); + + for (int i = 0; i < 5; i++) { + KeyValue enrichedOrderKeyValue = enrichedOrders.get(i); + assertThat(enrichedOrderKeyValue.key == i).isTrue(); + EnrichedOrder enrichedOrder = enrichedOrderKeyValue.value; + assertThat(enrichedOrder.getOrder().customerId == i).isTrue(); + assertThat(enrichedOrder.getOrder().productId == i).isTrue(); + assertThat(enrichedOrder.getCustomer().name.equals("customer-" + i)).isTrue(); + assertThat(enrichedOrder.getProduct().name.equals("product-" + i)).isTrue(); + } + pfCustomer.destroy(); + pfProduct.destroy(); + pfOrder.destroy(); + consumer.close(); + } + } + + @Test + public void testTimeExtractor() throws Exception { + SpringApplication app = new SpringApplication(OrderEnricherApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.function.definition=forTimeExtractorTest", + "--spring.cloud.stream.bindings.forTimeExtractorTest-in-0.destination=orders", + "--spring.cloud.stream.bindings.forTimeExtractorTest-in-1.destination=customers", + "--spring.cloud.stream.bindings.forTimeExtractorTest-in-2.destination=products", + "--spring.cloud.stream.bindings.forTimeExtractorTest-out-0.destination=enriched-order", + "--spring.cloud.stream.kafka.streams.bindings.forTimeExtractorTest-in-0.consumer.timestampExtractorBeanName" + + "=timestampExtractor", + "--spring.cloud.stream.kafka.streams.bindings.forTimeExtractorTest-in-1.consumer.timestampExtractorBeanName" + + "=timestampExtractor", + "--spring.cloud.stream.kafka.streams.bindings.forTimeExtractorTest-in-2.consumer.timestampExtractorBeanName" + + "=timestampExtractor", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=10000", + "--spring.cloud.stream.kafka.streams.bindings.order.consumer.applicationId=" + + "testTimeExtractor-abc", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + + final KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties = + context.getBean(KafkaStreamsExtendedBindingProperties.class); + + final Map bindings = kafkaStreamsExtendedBindingProperties.getBindings(); + + final KafkaStreamsBindingProperties kafkaStreamsBindingProperties0 = bindings.get("forTimeExtractorTest-in-0"); + final String timestampExtractorBeanName0 = kafkaStreamsBindingProperties0.getConsumer().getTimestampExtractorBeanName(); + final TimestampExtractor timestampExtractor0 = context.getBean(timestampExtractorBeanName0, TimestampExtractor.class); + assertThat(timestampExtractor0).isNotNull(); + + final KafkaStreamsBindingProperties kafkaStreamsBindingProperties1 = bindings.get("forTimeExtractorTest-in-1"); + final String timestampExtractorBeanName1 = kafkaStreamsBindingProperties1.getConsumer().getTimestampExtractorBeanName(); + final TimestampExtractor timestampExtractor1 = context.getBean(timestampExtractorBeanName1, TimestampExtractor.class); + assertThat(timestampExtractor1).isNotNull(); + + final KafkaStreamsBindingProperties kafkaStreamsBindingProperties2 = bindings.get("forTimeExtractorTest-in-2"); + final String timestampExtractorBeanName2 = kafkaStreamsBindingProperties2.getConsumer().getTimestampExtractorBeanName(); + final TimestampExtractor timestampExtractor2 = context.getBean(timestampExtractorBeanName2, TimestampExtractor.class); + assertThat(timestampExtractor2).isNotNull(); + } + } + + @EnableAutoConfiguration + public static class OrderEnricherApplication { + + @Bean + public Function, + Function, + Function, KStream>>> process() { + + return orderStream -> ( + customers -> ( + products -> ( + orderStream.join(customers, + (orderId, order) -> order.getCustomerId(), + (order, customer) -> new CustomerOrder(customer, order)) + .join(products, + (orderId, customerOrder) -> customerOrder + .productId(), + (customerOrder, product) -> { + EnrichedOrder enrichedOrder = new EnrichedOrder(); + enrichedOrder.setProduct(product); + enrichedOrder.setCustomer(customerOrder.customer); + enrichedOrder.setOrder(customerOrder.order); + return enrichedOrder; + }) + ) + ) + ); + } + + @Bean + public Function, + Function, + Function, KStream>>> forTimeExtractorTest() { + return orderStream -> + customers -> + products -> orderStream; + } + + @Bean + public TimestampExtractor timestampExtractor() { + return new WallclockTimestampExtractor(); + } + } + + static class Order { + + long customerId; + long productId; + + public long getCustomerId() { + return customerId; + } + + public void setCustomerId(long customerId) { + this.customerId = customerId; + } + + public long getProductId() { + return productId; + } + + public void setProductId(long productId) { + this.productId = productId; + } + } + + static class Customer { + + String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + + static class Product { + + String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + + static class EnrichedOrder { + + Product product; + Customer customer; + Order order; + + public Product getProduct() { + return product; + } + + public void setProduct(Product product) { + this.product = product; + } + + public Customer getCustomer() { + return customer; + } + + public void setCustomer(Customer customer) { + this.customer = customer; + } + + public Order getOrder() { + return order; + } + + public void setOrder(Order order) { + this.order = order; + } + } + + private static class CustomerOrder { + private final Customer customer; + private final Order order; + + CustomerOrder(final Customer customer, final Order order) { + this.customer = customer; + this.order = order; + } + + long productId() { + return order.getProductId(); + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/StreamToTableJoinFunctionTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/StreamToTableJoinFunctionTests.java new file mode 100644 index 000000000..6f269fa11 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/function/StreamToTableJoinFunctionTests.java @@ -0,0 +1,579 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.function; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.LongDeserializer; +import org.apache.kafka.common.serialization.LongSerializer; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.JoinWindows; +import org.apache.kafka.streams.kstream.Joined; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.StreamJoined; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.stream.binder.Binder; +import org.springframework.cloud.stream.binder.BinderFactory; +import org.springframework.cloud.stream.binder.ConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder; +import org.springframework.cloud.stream.binder.ProducerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsProducerProperties; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.CleanupConfig; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.util.Assert; + +import static org.assertj.core.api.Assertions.assertThat; + +public class StreamToTableJoinFunctionTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, + true, "output-topic-1", "output-topic-2", "user-clicks-2", "user-regions-2"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule.getEmbeddedKafka(); + + @Test + public void testStreamToTable() { + SpringApplication app = new SpringApplication(CountClicksPerRegionApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + Consumer consumer; + Map consumerProps = KafkaTestUtils.consumerProps("group-1", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, LongDeserializer.class); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "output-topic-1"); + + runTest(app, consumer); + } + + @Test + public void testStreamToTableBiFunction() { + SpringApplication app = new SpringApplication(BiFunctionCountClicksPerRegionApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + Consumer consumer; + Map consumerProps = KafkaTestUtils.consumerProps("group-2", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, LongDeserializer.class); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "output-topic-1"); + + runTest(app, consumer); + } + + @Test + public void testStreamToTableBiConsumer() throws Exception { + SpringApplication app = new SpringApplication(BiConsumerApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + Consumer consumer; + Map consumerProps = KafkaTestUtils.consumerProps("group-2", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, LongDeserializer.class); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "output-topic-1"); + + try (ConfigurableApplicationContext ignored = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.process-in-0.destination=user-clicks-1", + "--spring.cloud.stream.bindings.process-in-1.destination=user-regions-1", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=10000", + "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.applicationId" + + "=testStreamToTableBiConsumer", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + + // Input 1: Region per user (multiple records allowed per user). + List> userRegions = Arrays.asList( + new KeyValue<>("alice", "asia") + ); + + Map senderProps1 = KafkaTestUtils.producerProps(embeddedKafka); + senderProps1.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + senderProps1.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + + DefaultKafkaProducerFactory pf1 = new DefaultKafkaProducerFactory<>(senderProps1); + KafkaTemplate template1 = new KafkaTemplate<>(pf1, true); + template1.setDefaultTopic("user-regions-1"); + + for (KeyValue keyValue : userRegions) { + template1.sendDefault(keyValue.key, keyValue.value); + } + + // Input 2: Clicks per user (multiple records allowed per user). + List> userClicks = Arrays.asList( + new KeyValue<>("alice", 13L) + ); + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + senderProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + senderProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, LongSerializer.class); + + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("user-clicks-1"); + + for (KeyValue keyValue : userClicks) { + template.sendDefault(keyValue.key, keyValue.value); + } + + Assert.isTrue(BiConsumerApplication.latch.await(10, TimeUnit.SECONDS), "Failed to receive message"); + + } + finally { + consumer.close(); + } + } + + private void runTest(SpringApplication app, Consumer consumer) { + try (ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.process-in-0.destination=user-clicks-1", + "--spring.cloud.stream.bindings.process-in-1.destination=user-regions-1", + "--spring.cloud.stream.bindings.process-out-0.destination=output-topic-1", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=10000", + "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.applicationId" + + "=StreamToTableJoinFunctionTests-abc", + "--spring.cloud.stream.kafka.streams.bindings.process-in-1.consumer.topic.properties.cleanup.policy=compact", + "--spring.cloud.stream.kafka.streams.bindings.process-out-0.producer.topic.properties.cleanup.policy=compact", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + + // Input 1: Region per user (multiple records allowed per user). + List> userRegions = Arrays.asList( + new KeyValue<>("alice", "asia"), /* Alice lived in Asia originally... */ + new KeyValue<>("bob", "americas"), + new KeyValue<>("chao", "asia"), + new KeyValue<>("dave", "europe"), + new KeyValue<>("alice", "europe"), /* ...but moved to Europe some time later. */ + new KeyValue<>("eve", "americas"), + new KeyValue<>("fang", "asia") + ); + + Map senderProps1 = KafkaTestUtils.producerProps(embeddedKafka); + senderProps1.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + senderProps1.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + + DefaultKafkaProducerFactory pf1 = new DefaultKafkaProducerFactory<>(senderProps1); + KafkaTemplate template1 = new KafkaTemplate<>(pf1, true); + template1.setDefaultTopic("user-regions-1"); + + for (KeyValue keyValue : userRegions) { + template1.sendDefault(keyValue.key, keyValue.value); + } + + // Input 2: Clicks per user (multiple records allowed per user). + List> userClicks = Arrays.asList( + new KeyValue<>("alice", 13L), + new KeyValue<>("bob", 4L), + new KeyValue<>("chao", 25L), + new KeyValue<>("bob", 19L), + new KeyValue<>("dave", 56L), + new KeyValue<>("eve", 78L), + new KeyValue<>("alice", 40L), + new KeyValue<>("fang", 99L) + ); + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + senderProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + senderProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, LongSerializer.class); + + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("user-clicks-1"); + + for (KeyValue keyValue : userClicks) { + template.sendDefault(keyValue.key, keyValue.value); + } + + List> expectedClicksPerRegion = Arrays.asList( + new KeyValue<>("americas", 101L), + new KeyValue<>("europe", 109L), + new KeyValue<>("asia", 124L) + ); + + //Verify that we receive the expected data + int count = 0; + long start = System.currentTimeMillis(); + List> actualClicksPerRegion = new ArrayList<>(); + do { + ConsumerRecords records = KafkaTestUtils.getRecords(consumer); + count = count + records.count(); + for (ConsumerRecord record : records) { + actualClicksPerRegion.add(new KeyValue<>(record.key(), record.value())); + } + } while (count < expectedClicksPerRegion.size() && (System.currentTimeMillis() - start) < 30000); + + assertThat(count == expectedClicksPerRegion.size()).isTrue(); + assertThat(actualClicksPerRegion).hasSameElementsAs(expectedClicksPerRegion); + + // Testing certain ancillary configuration of GlobalKTable around topics creation. + // See this issue: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/687 + BinderFactory binderFactory = context.getBeanFactory() + .getBean(BinderFactory.class); + + Binder ktableBinder = binderFactory + .getBinder("ktable", KTable.class); + + KafkaStreamsConsumerProperties inputX = (KafkaStreamsConsumerProperties) ((ExtendedPropertiesBinder) ktableBinder) + .getExtendedConsumerProperties("process-in-1"); + String cleanupPolicyX = inputX.getTopic().getProperties().get("cleanup.policy"); + + assertThat(cleanupPolicyX).isEqualTo("compact"); + + Binder kStreamBinder = binderFactory + .getBinder("kstream", KStream.class); + + KafkaStreamsProducerProperties producerProperties = (KafkaStreamsProducerProperties) ((ExtendedPropertiesBinder) kStreamBinder) + .getExtendedProducerProperties("process-out-0"); + + String cleanupPolicyOutput = producerProperties.getTopic().getProperties().get("cleanup.policy"); + + assertThat(cleanupPolicyOutput).isEqualTo("compact"); + } + finally { + consumer.close(); + } + } + +// @Test + public void testGlobalStartOffsetWithLatestAndIndividualBindingWthEarliest() throws Exception { + SpringApplication app = new SpringApplication(BiFunctionCountClicksPerRegionApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + Consumer consumer; + Map consumerProps = KafkaTestUtils.consumerProps("group-3", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, LongDeserializer.class); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "output-topic-2"); + + // Produce data first to the input topic to test the startOffset setting on the + // binding (which is set to earliest below). + // Input 1: Clicks per user (multiple records allowed per user). + List> userClicks = Arrays.asList( + new KeyValue<>("alice", 100L), + new KeyValue<>("alice", 100L), + new KeyValue<>("alice", 100L), + new KeyValue<>("alice", 100L), + new KeyValue<>("alice", 100L), + new KeyValue<>("alice", 100L), + new KeyValue<>("alice", 100L), + new KeyValue<>("alice", 100L), + new KeyValue<>("alice", 100L), + new KeyValue<>("alice", 100L) + ); + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + senderProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + senderProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, LongSerializer.class); + + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("user-clicks-2"); + + for (KeyValue keyValue : userClicks) { + template.sendDefault(keyValue.key, keyValue.value); + } + + try (ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.process-in-0.destination=user-clicks-2", + "--spring.cloud.stream.bindings.process-in-1.destination=user-regions-2", + "--spring.cloud.stream.bindings.process-out-0.destination=output-topic-2", + "--spring.cloud.stream.bindings.process-in-0.consumer.useNativeDecoding=true", + "--spring.cloud.stream.bindings.process-in-1.consumer.useNativeDecoding=true", + "--spring.cloud.stream.bindings.process-out-0.producer.useNativeEncoding=true", + "--spring.cloud.stream.kafka.streams.binder.configuration.auto.offset.reset=latest", + "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.startOffset=earliest", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=10000", + "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.application-id" + + "=StreamToTableJoinFunctionTests-foobar", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString(), + "--spring.cloud.stream.kafka.streams.binder.zkNodes=" + embeddedKafka.getZookeeperConnectionString())) { + Thread.sleep(1000L); + + // Input 2: Region per user (multiple records allowed per user). + List> userRegions = Arrays.asList( + new KeyValue<>("alice", "asia"), /* Alice lived in Asia originally... */ + new KeyValue<>("bob", "americas"), + new KeyValue<>("chao", "asia"), + new KeyValue<>("dave", "europe"), + new KeyValue<>("alice", "europe"), /* ...but moved to Europe some time later. */ + new KeyValue<>("eve", "americas"), + new KeyValue<>("fang", "asia") + ); + + Map senderProps1 = KafkaTestUtils.producerProps(embeddedKafka); + senderProps1.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + senderProps1.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + + DefaultKafkaProducerFactory pf1 = new DefaultKafkaProducerFactory<>(senderProps1); + KafkaTemplate template1 = new KafkaTemplate<>(pf1, true); + template1.setDefaultTopic("user-regions-2"); + + for (KeyValue keyValue : userRegions) { + template1.sendDefault(keyValue.key, keyValue.value); + } + + + // Input 1: Clicks per user (multiple records allowed per user). + List> userClicks1 = Arrays.asList( + new KeyValue<>("bob", 4L), + new KeyValue<>("chao", 25L), + new KeyValue<>("bob", 19L), + new KeyValue<>("dave", 56L), + new KeyValue<>("eve", 78L), + new KeyValue<>("fang", 99L) + ); + + for (KeyValue keyValue : userClicks1) { + template.sendDefault(keyValue.key, keyValue.value); + } + + List> expectedClicksPerRegion = Arrays.asList( + new KeyValue<>("americas", 101L), + new KeyValue<>("europe", 56L), + new KeyValue<>("asia", 124L), + //1000 alice entries which were there in the topic before the consumer started. + //Since we set the startOffset to earliest for the topic, it will read them, + //but the join fails to associate with a valid region, thus UNKNOWN. + new KeyValue<>("UNKNOWN", 1000L) + ); + + //Verify that we receive the expected data + int count = 0; + long start = System.currentTimeMillis(); + List> actualClicksPerRegion = new ArrayList<>(); + do { + ConsumerRecords records = KafkaTestUtils.getRecords(consumer); + count = count + records.count(); + for (ConsumerRecord record : records) { + actualClicksPerRegion.add(new KeyValue<>(record.key(), record.value())); + } + } while (count < expectedClicksPerRegion.size() && (System.currentTimeMillis() - start) < 30000); + + // TODO: Matched count is 3 and not 4 (expectedClicksPerRegion.size()) when running with full suite. Investigate why. + // TODO: This behavior is only observed after the Spring Kafka upgrade to 2.5.0 and kafka client to 2.5. + // TODO: Note that the test passes fine as a single test. + assertThat(count).matches( + matchedCount -> matchedCount == expectedClicksPerRegion.size() - 1 || matchedCount == expectedClicksPerRegion.size()); + assertThat(actualClicksPerRegion).containsAnyElementsOf(expectedClicksPerRegion); + } + finally { + consumer.close(); + } + } + + @Test + public void testTrivialSingleKTableInputAsNonDeclarative() { + SpringApplication app = new SpringApplication(TrivialKTableApp.class); + app.setWebApplicationType(WebApplicationType.NONE); + app.run("--server.port=0", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString(), + "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.application-id=" + + "testTrivialSingleKTableInputAsNonDeclarative"); + //All we are verifying is that this application didn't throw any errors. + //See this issue: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/536 + } + + @Test + public void testTwoKStreamsCanBeJoined() { + SpringApplication app = new SpringApplication( + JoinProcessor.class); + app.setWebApplicationType(WebApplicationType.NONE); + app.run("--server.port=0", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString(), + "--spring.application.name=" + + "two-kstream-input-join-integ-test"); + //All we are verifying is that this application didn't throw any errors. + //See this issue: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/701 + } + + + /** + * Tuple for a region and its associated number of clicks. + */ + private static final class RegionWithClicks { + + private final String region; + private final long clicks; + + RegionWithClicks(String region, long clicks) { + if (region == null || region.isEmpty()) { + throw new IllegalArgumentException("region must be set"); + } + if (clicks < 0) { + throw new IllegalArgumentException("clicks must not be negative"); + } + this.region = region; + this.clicks = clicks; + } + + public String getRegion() { + return region; + } + + public long getClicks() { + return clicks; + } + + } + + @EnableAutoConfiguration + public static class CountClicksPerRegionApplication { + + @Bean + public Function, Function, KStream>> process() { + return userClicksStream -> (userRegionsTable -> (userClicksStream + .leftJoin(userRegionsTable, (clicks, region) -> new RegionWithClicks(region == null ? + "UNKNOWN" : region, clicks), + Joined.with(Serdes.String(), Serdes.Long(), null)) + .map((user, regionWithClicks) -> new KeyValue<>(regionWithClicks.getRegion(), + regionWithClicks.getClicks())) + .groupByKey(Grouped.with(Serdes.String(), Serdes.Long())) + .reduce(Long::sum, Materialized.as("CountClicks-" + UUID.randomUUID())) + .toStream())); + } + + @Bean + public CleanupConfig cleanupConfig() { + return new CleanupConfig(false, true); + } + } + + @EnableAutoConfiguration + public static class BiFunctionCountClicksPerRegionApplication { + + @Bean + public BiFunction, KTable, KStream> process() { + return (userClicksStream, userRegionsTable) -> (userClicksStream + .leftJoin(userRegionsTable, (clicks, region) -> new RegionWithClicks(region == null ? + "UNKNOWN" : region, clicks), + Joined.with(Serdes.String(), Serdes.Long(), null)) + .map((user, regionWithClicks) -> new KeyValue<>(regionWithClicks.getRegion(), + regionWithClicks.getClicks())) + .groupByKey(Grouped.with(Serdes.String(), Serdes.Long())) + .reduce(Long::sum, Materialized.as("CountClicks-" + UUID.randomUUID())) + .toStream()); + } + + @Bean + public CleanupConfig cleanupConfig() { + return new CleanupConfig(false, true); + } + } + + @EnableAutoConfiguration + public static class BiConsumerApplication { + + static CountDownLatch latch = new CountDownLatch(2); + + @Bean + public BiConsumer, KTable> process() { + return (userClicksStream, userRegionsTable) -> { + userClicksStream.foreach((key, value) -> latch.countDown()); + userRegionsTable.toStream().foreach((key, value) -> latch.countDown()); + }; + } + } + + @EnableAutoConfiguration + public static class TrivialKTableApp { + + public java.util.function.Consumer> process() { + return inputTable -> inputTable.toStream().foreach((key, value) -> System.out.println("key : value " + key + " : " + value)); + } + } + + @EnableAutoConfiguration + public static class JoinProcessor { + + public BiConsumer, KStream> testProcessor() { + return (input1Stream, input2Stream) -> input1Stream + .join(input2Stream, + (event1, event2) -> null, + JoinWindows.of(Duration.ofMillis(5)), + StreamJoined.with( + Serdes.String(), + Serdes.String(), + Serdes.String() + ) + ); + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializationErrorHandlerByKafkaTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializationErrorHandlerByKafkaTests.java new file mode 100644 index 000000000..a2323c555 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializationErrorHandlerByKafkaTests.java @@ -0,0 +1,271 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Map; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; +import org.springframework.cloud.stream.binder.kafka.utils.DlqPartitionFunction; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.PropertySource; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.messaging.handler.annotation.SendTo; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +/** + * @author Soby Chacko + */ +@RunWith(SpringRunner.class) +@ContextConfiguration +@DirtiesContext +public abstract class DeserializationErrorHandlerByKafkaTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "abc-DeserializationErrorHandlerByKafkaTests-In", + "xyz-DeserializationErrorHandlerByKafkaTests-In", + "DeserializationErrorHandlerByKafkaTests-out", + "error.abc-DeserializationErrorHandlerByKafkaTests-In.group", + "error.xyz-DeserializationErrorHandlerByKafkaTests-In.group", + "error.word1.groupx", + "error.word2.groupx"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + @SpyBean + org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsMessageConversionDelegate conversionDelegate; + + private static Consumer consumer; + + @BeforeClass + public static void setUp() { + System.setProperty("spring.cloud.stream.kafka.streams.binder.brokers", + embeddedKafka.getBrokersAsString()); + + System.setProperty("server.port", "0"); + System.setProperty("spring.jmx.enabled", "false"); + + Map consumerProps = KafkaTestUtils.consumerProps("fooc", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer, "DeserializationErrorHandlerByKafkaTests-out", "DeserializationErrorHandlerByKafkaTests-out"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + System.clearProperty("spring.cloud.stream.kafka.streams.binder.brokers"); + System.clearProperty("server.port"); + System.clearProperty("spring.jmx.enabled"); + } + + @SpringBootTest(properties = { + "spring.cloud.stream.bindings.input.destination=abc-DeserializationErrorHandlerByKafkaTests-In", + "spring.cloud.stream.bindings.output.destination=DeserializationErrorHandlerByKafkaTests-Out", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=deser-kafka-dlq", + "spring.cloud.stream.bindings.input.group=group", + "spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler=sendToDlq", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.valueSerde=" + + "org.apache.kafka.common.serialization.Serdes$IntegerSerde" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) + public static class DeserializationByKafkaAndDlqTests + extends DeserializationErrorHandlerByKafkaTests { + + @Test + @Ignore + public void test() { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("abc-DeserializationErrorHandlerByKafkaTests-In"); + template.sendDefault(1, null, "foobar"); + + Map consumerProps = KafkaTestUtils.consumerProps("foobar", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + Consumer consumer1 = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer1, "error.abc-DeserializationErrorHandlerByKafkaTests-In.group"); + + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer1, + "error.abc-DeserializationErrorHandlerByKafkaTests-In.group"); + assertThat(cr.value()).isEqualTo("foobar"); + assertThat(cr.partition()).isEqualTo(0); // custom partition function + + // Ensuring that the deserialization was indeed done by Kafka natively + verify(conversionDelegate, never()).deserializeOnInbound(any(Class.class), + any(KStream.class)); + verify(conversionDelegate, never()).serializeOnOutbound(any(KStream.class)); + } + + } + + @SpringBootTest(properties = { + "spring.cloud.stream.bindings.input.destination=xyz-DeserializationErrorHandlerByKafkaTests-In", + "spring.cloud.stream.bindings.output.destination=DeserializationErrorHandlerByKafkaTests-Out", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=deser-kafka-dlq", + "spring.cloud.stream.bindings.input.group=group", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.deserializationExceptionHandler=sendToDlq", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.valueSerde=" + + "org.apache.kafka.common.serialization.Serdes$IntegerSerde" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) + public static class DeserializationByKafkaAndDlqPerBindingTests + extends DeserializationErrorHandlerByKafkaTests { + + @Test + public void test() { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("xyz-DeserializationErrorHandlerByKafkaTests-In"); + template.sendDefault(1, null, "foobar"); + + Map consumerProps = KafkaTestUtils.consumerProps("foobar", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + Consumer consumer1 = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer1, "error.xyz-DeserializationErrorHandlerByKafkaTests-In.group"); + + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer1, + "error.xyz-DeserializationErrorHandlerByKafkaTests-In.group"); + assertThat(cr.value()).isEqualTo("foobar"); + assertThat(cr.partition()).isEqualTo(0); // custom partition function + + // Ensuring that the deserialization was indeed done by Kafka natively + verify(conversionDelegate, never()).deserializeOnInbound(any(Class.class), + any(KStream.class)); + verify(conversionDelegate, never()).serializeOnOutbound(any(KStream.class)); + } + + } + + @SpringBootTest(properties = { + "spring.cloud.stream.bindings.input.destination=word1,word2", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=deser-kafka-dlq-multi-input", + "spring.cloud.stream.bindings.input.group=groupx", + "spring.cloud.stream.kafka.streams.binder.serdeError=sendToDlq", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.valueSerde=" + + "org.apache.kafka.common.serialization.Serdes$IntegerSerde" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) + // @checkstyle:on + public static class DeserializationByKafkaAndDlqTestsWithMultipleInputs + extends DeserializationErrorHandlerByKafkaTests { + + @Test + public void test() { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("word1"); + template.sendDefault("foobar"); + + template.setDefaultTopic("word2"); + template.sendDefault("foobar"); + + Map consumerProps = KafkaTestUtils.consumerProps("foobarx", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + Consumer consumer1 = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer1, "error.word1.groupx", + "error.word2.groupx"); + + ConsumerRecord cr1 = KafkaTestUtils.getSingleRecord(consumer1, + "error.word1.groupx"); + assertThat(cr1.value()).isEqualTo("foobar"); + ConsumerRecord cr2 = KafkaTestUtils.getSingleRecord(consumer1, + "error.word2.groupx"); + assertThat(cr2.value()).isEqualTo("foobar"); + + // Ensuring that the deserialization was indeed done by Kafka natively + verify(conversionDelegate, never()).deserializeOnInbound(any(Class.class), + any(KStream.class)); + verify(conversionDelegate, never()).serializeOnOutbound(any(KStream.class)); + } + + } + + @EnableBinding(KafkaStreamsProcessor.class) + @EnableAutoConfiguration + @PropertySource("classpath:/org/springframework/cloud/stream/binder/kstream/integTest-1.properties") + public static class WordCountProcessorApplication { + + @StreamListener("input") + @SendTo("output") + public KStream process(KStream input) { + + return input + .flatMapValues( + value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .map((key, value) -> new KeyValue<>(value, value)) + .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) + .windowedBy(TimeWindows.of(Duration.ofMillis(5000))).count(Materialized.as("foo-WordCounts-x")) + .toStream().map((key, value) -> new KeyValue<>(null, + "Count for " + key.key() + " : " + value)); + } + + @Bean + public DlqPartitionFunction partitionFunction() { + return (group, rec, ex) -> 0; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializtionErrorHandlerByBinderTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializtionErrorHandlerByBinderTests.java new file mode 100644 index 000000000..bf2d9cfce --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DeserializtionErrorHandlerByBinderTests.java @@ -0,0 +1,286 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration; + +import java.time.Duration; +import java.util.Map; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.serializer.JsonSerde; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.messaging.handler.annotation.SendTo; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; + +/** + * @author Soby Chacko + */ +@RunWith(SpringRunner.class) +@ContextConfiguration +@DirtiesContext +public abstract class DeserializtionErrorHandlerByBinderTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "foos", "goos", + "counts-id", "error.foos.foobar-group", "error.goos.foobar-group", "error.foos1.fooz-group", + "error.foos2.fooz-group"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + @SpyBean + org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsMessageConversionDelegate conversionDelegate; + + private static Consumer consumer; + + @BeforeClass + public static void setUp() throws Exception { + System.setProperty("spring.cloud.stream.kafka.streams.binder.brokers", + embeddedKafka.getBrokersAsString()); + System.setProperty("server.port", "0"); + System.setProperty("spring.jmx.enabled", "false"); + + Map consumerProps = KafkaTestUtils.consumerProps("kafka-streams-dlq-tests", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "counts-id"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + System.clearProperty("spring.cloud.stream.kafka.streams.binder.brokers"); + System.clearProperty("server.port"); + System.clearProperty("spring.jmx.enabled"); + } + + @SpringBootTest(properties = { + "spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", + "spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", + "spring.cloud.stream.bindings.input.destination=foos", + "spring.cloud.stream.bindings.output.destination=counts-id", + "spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$IntegerSerde", + "spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "spring.cloud.stream.kafka.streams.binder.deserializationExceptionHandler=sendToDlq", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id" + + "=deserializationByBinderAndDlqTests", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.dlqPartitions=1", + "spring.cloud.stream.bindings.input.group=foobar-group" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) + public static class DeserializationByBinderAndDlqTests + extends DeserializtionErrorHandlerByBinderTests { + + @Test + @Ignore + public void test() { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("foos"); + template.sendDefault(1, 7, "hello"); + + Map consumerProps = KafkaTestUtils.consumerProps("foobar", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + Consumer consumer1 = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer1, + "error.foos.foobar-group"); + + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer1, + "error.foos.foobar-group"); + assertThat(cr.value()).isEqualTo("hello"); + assertThat(cr.partition()).isEqualTo(0); + + // Ensuring that the deserialization was indeed done by the binder + verify(conversionDelegate).deserializeOnInbound(any(Class.class), + any(KStream.class)); + } + } + + @SpringBootTest(properties = { + "spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", + "spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", + "spring.cloud.stream.bindings.input.destination=goos", + "spring.cloud.stream.bindings.output.destination=counts-id", + "spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$IntegerSerde", + "spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.deserializationExceptionHandler=sendToDlq", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id" + + "=deserializationByBinderAndDlqTests", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.dlqPartitions=1", + "spring.cloud.stream.bindings.input.group=foobar-group" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) + public static class DeserializationByBinderAndDlqSetOnConsumerBindingTests + extends DeserializtionErrorHandlerByBinderTests { + + @Test + public void test() { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("goos"); + template.sendDefault(1, 7, "hello"); + + Map consumerProps = KafkaTestUtils.consumerProps("foobar", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + Consumer consumer1 = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer1, + "error.goos.foobar-group"); + + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer1, + "error.goos.foobar-group"); + assertThat(cr.value()).isEqualTo("hello"); + assertThat(cr.partition()).isEqualTo(0); + + // Ensuring that the deserialization was indeed done by the binder + verify(conversionDelegate).deserializeOnInbound(any(Class.class), + any(KStream.class)); + } + } + + @SpringBootTest(properties = { + "spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", + "spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", + "spring.cloud.stream.bindings.input.destination=foos1,foos2", + "spring.cloud.stream.bindings.output.destination=counts-id", + "spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "spring.cloud.stream.kafka.streams.binder.serdeError=sendToDlq", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id" + + "=deserializationByBinderAndDlqTestsWithMultipleInputs", + "spring.cloud.stream.bindings.input.group=fooz-group" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) + public static class DeserializationByBinderAndDlqTestsWithMultipleInputs + extends DeserializtionErrorHandlerByBinderTests { + + @Test + @SuppressWarnings("unchecked") + public void test() { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("foos1"); + template.sendDefault("hello"); + + template.setDefaultTopic("foos2"); + template.sendDefault("hello"); + + Map consumerProps = KafkaTestUtils.consumerProps("foobar1", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + Consumer consumer1 = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer1, "error.foos1.fooz-group", + "error.foos2.fooz-group"); + + ConsumerRecord cr1 = KafkaTestUtils.getSingleRecord(consumer1, + "error.foos1.fooz-group"); + assertThat(cr1.value().equals("hello")).isTrue(); + + ConsumerRecord cr2 = KafkaTestUtils.getSingleRecord(consumer1, + "error.foos2.fooz-group"); + assertThat(cr2.value().equals("hello")).isTrue(); + + // Ensuring that the deserialization was indeed done by the binder + verify(conversionDelegate).deserializeOnInbound(any(Class.class), + any(KStream.class)); + } + + } + + @EnableBinding(KafkaStreamsProcessor.class) + @EnableAutoConfiguration + public static class ProductCountApplication { + + @StreamListener("input") + @SendTo("output") + public KStream process(KStream input) { + return input.filter((key, product) -> product.getId() == 123) + .map((key, value) -> new KeyValue<>(value, value)) + .groupByKey(Grouped.with(new JsonSerde<>(Product.class), + new JsonSerde<>(Product.class))) + .windowedBy(TimeWindows.of(Duration.ofMillis(5000))) + .count(Materialized.as("id-count-store-x")).toStream() + .map((key, value) -> new KeyValue<>(key.key().id, value)); + } + + } + + static class Product { + + Integer id; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DlqDestinationResolverTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DlqDestinationResolverTests.java new file mode 100644 index 000000000..e45bc639a --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/DlqDestinationResolverTests.java @@ -0,0 +1,149 @@ +/* + * Copyright 2020-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Map; +import java.util.function.Function; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsBindingInformationCatalogue; +import org.springframework.cloud.stream.binder.kafka.utils.DlqDestinationResolver; +import org.springframework.cloud.stream.binder.kafka.utils.DlqPartitionFunction; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Soby Chacko + */ +public class DlqDestinationResolverTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "topic1-dlq", + "topic2-dlq"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + @Test + public void testDlqDestinationResolverWorks() throws Exception { + SpringApplication app = new SpringApplication(WordCountProcessorApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run( + "--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.function.definition=process", + "--spring.cloud.stream.bindings.process-in-0.destination=word1,word2", + "--spring.cloud.stream.bindings.process-out-0.destination=test-output", + "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.application-id=dlq-dest-resolver-test", + "--spring.cloud.stream.kafka.streams.binder.serdeError=sendToDlq", + "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.valueSerde=" + + "org.apache.kafka.common.serialization.Serdes$IntegerSerde", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("word1"); + template.sendDefault("foobar"); + + template.setDefaultTopic("word2"); + template.sendDefault("foobar"); + + Map consumerProps = KafkaTestUtils.consumerProps("some-random-group", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + Consumer consumer1 = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer1, "topic1-dlq", + "topic2-dlq"); + + ConsumerRecord cr1 = KafkaTestUtils.getSingleRecord(consumer1, + "topic1-dlq"); + assertThat(cr1.value()).isEqualTo("foobar"); + ConsumerRecord cr2 = KafkaTestUtils.getSingleRecord(consumer1, + "topic2-dlq"); + assertThat(cr2.value()).isEqualTo("foobar"); + + final KafkaStreamsBindingInformationCatalogue catalogue = context.getBean(KafkaStreamsBindingInformationCatalogue.class); + assertThat(catalogue.getDlqProducerFactories().size()).isEqualTo(1); + } + finally { + pf.destroy(); + } + } + } + + @EnableAutoConfiguration + public static class WordCountProcessorApplication { + + @Bean + public Function, KStream> process() { + + return input -> input + .flatMapValues( + value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .map((key, value) -> new KeyValue<>(value, value)) + .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) + .windowedBy(TimeWindows.of(Duration.ofSeconds(5))).count(Materialized.as("foo-WordCounts-x")) + .toStream().map((key, value) -> new KeyValue<>(null, + "Count for " + key.key() + " : " + value)); + } + + @Bean + public DlqPartitionFunction partitionFunction() { + return (group, rec, ex) -> 0; + } + + @Bean + public DlqDestinationResolver dlqDestinationResolver() { + return (rec, ex) -> { + if (rec.topic().equals("word1")) { + return "topic1-dlq"; + } + return "topic2-dlq"; + }; + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderDestinationIsPatternTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderDestinationIsPatternTests.java new file mode 100644 index 000000000..47004a293 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderDestinationIsPatternTests.java @@ -0,0 +1,111 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration; + +import java.util.Map; +import java.util.function.Function; + +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.streams.kstream.KStream; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Michael Stoettinger + */ +public class KafkaStreamsBinderDestinationIsPatternTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "in.1", "in.2", "out"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + private static org.apache.kafka.clients.consumer.Consumer consumer; + + @BeforeClass + public static void setUp() { + Map consumerProps = KafkaTestUtils.consumerProps("group", "true", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer, "out"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + } + + @Test + public void test() { + SpringApplication app = new SpringApplication(ConsumingApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.cloud.stream.bindings.process-out-0.destination=out", + "--spring.cloud.stream.bindings.process-in-0.destination=in.*", + "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.destinationIsPattern=true", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString()); + try { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory producerFactory = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(producerFactory, true); + + // send message to both topics that fit the pattern + template.send("in.1", "foo1"); + assertThat(KafkaTestUtils.getSingleRecord(consumer, "out").value()) + .isEqualTo("foo1"); + template.send("in.2", "foo2"); + assertThat(KafkaTestUtils.getSingleRecord(consumer, "out").value()) + .isEqualTo("foo2"); + } + finally { + context.close(); + } + } + + @EnableAutoConfiguration + public static class ConsumingApplication { + + @Bean + public Function, KStream> process() { + return input -> input; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderHealthIndicatorTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderHealthIndicatorTests.java new file mode 100644 index 000000000..a0694b152 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderHealthIndicatorTests.java @@ -0,0 +1,327 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler; +import org.apache.kafka.streams.kstream.KStream; +import org.assertj.core.util.Lists; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.actuate.health.CompositeHealthContributor; +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.Status; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.Output; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsBinderHealthIndicator; +import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.config.KafkaStreamsCustomizer; +import org.springframework.kafka.config.StreamsBuilderFactoryBeanConfigurer; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.SendResult; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.messaging.handler.annotation.SendTo; +import org.springframework.util.concurrent.ListenableFuture; +import org.springframework.util.concurrent.ListenableFutureCallback; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Arnaud Jardiné + */ +public class KafkaStreamsBinderHealthIndicatorTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "out", "out2"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + @BeforeClass + public static void setUp() { + System.setProperty("logging.level.org.apache.kafka", "OFF"); + } + + @Test + public void healthIndicatorUpTest() throws Exception { + try (ConfigurableApplicationContext context = singleStream("ApplicationHealthTest-xyz")) { + receive(context, + Lists.newArrayList(new ProducerRecord<>("in", "{\"id\":\"123\"}"), + new ProducerRecord<>("in", "{\"id\":\"123\"}")), + Status.UP, "out"); + } + } + + @Test + public void healthIndicatorUpMultipleCallsTest() throws Exception { + try (ConfigurableApplicationContext context = singleStream("ApplicationHealthTest-xyz")) { + int callsToPerform = 5; + for (int i = 0; i < callsToPerform; i++) { + receive(context, + Lists.newArrayList(new ProducerRecord<>("in", "{\"id\":\"123\"}"), + new ProducerRecord<>("in", "{\"id\":\"123\"}")), + Status.UP, "out"); + } + } + } + + @Test + public void healthIndicatorDownTest() throws Exception { + try (ConfigurableApplicationContext context = singleStream("ApplicationHealthTest-xyzabc")) { + receive(context, + Lists.newArrayList(new ProducerRecord<>("in", "{\"id\":\"123\"}"), + new ProducerRecord<>("in", "{\"id\":\"124\"}")), + Status.DOWN, "out"); + } + } + + @Test + public void healthIndicatorUpMultipleKStreamsTest() throws Exception { + try (ConfigurableApplicationContext context = multipleStream()) { + receive(context, + Lists.newArrayList(new ProducerRecord<>("in", "{\"id\":\"123\"}"), + new ProducerRecord<>("in2", "{\"id\":\"123\"}")), + Status.UP, "out", "out2"); + } + } + + @Test + public void healthIndicatorDownMultipleKStreamsTest() throws Exception { + try (ConfigurableApplicationContext context = multipleStream()) { + receive(context, + Lists.newArrayList(new ProducerRecord<>("in", "{\"id\":\"123\"}"), + new ProducerRecord<>("in2", "{\"id\":\"124\"}")), + Status.DOWN, "out", "out2"); + } + } + + private static boolean waitFor(Status status, Map details) { + if (status == Status.UP) { + String threadState = (String) details.get("threadState"); + return threadState != null + && (threadState.equalsIgnoreCase(KafkaStreams.State.REBALANCING.name()) + || threadState.equalsIgnoreCase("PARTITIONS_REVOKED") + || threadState.equalsIgnoreCase("PARTITIONS_ASSIGNED") + || threadState.equalsIgnoreCase( + KafkaStreams.State.PENDING_SHUTDOWN.name())); + } + return false; + } + + private void receive(ConfigurableApplicationContext context, + List> records, Status expected, + String... topics) throws Exception { + Map consumerProps = KafkaTestUtils.consumerProps("group-id0", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + try (Consumer consumer = cf.createConsumer()) { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + CountDownLatch latch = new CountDownLatch(records.size()); + for (ProducerRecord record : records) { + ListenableFuture> future = template + .send(record); + future.addCallback( + new ListenableFutureCallback>() { + @Override + public void onFailure(Throwable ex) { + Assert.fail(); + } + + @Override + public void onSuccess(SendResult result) { + latch.countDown(); + } + }); + } + + latch.await(5, TimeUnit.SECONDS); + + embeddedKafka.consumeFromEmbeddedTopics(consumer, topics); + KafkaTestUtils.getRecords(consumer, 1000); + + TimeUnit.SECONDS.sleep(5); + checkHealth(context, expected); + } + finally { + pf.destroy(); + } + } + + private static void checkHealth(ConfigurableApplicationContext context, + Status expected) throws InterruptedException { + CompositeHealthContributor healthIndicator = context + .getBean("bindersHealthContributor", CompositeHealthContributor.class); + KafkaStreamsBinderHealthIndicator kafkaStreamsBinderHealthIndicator = (KafkaStreamsBinderHealthIndicator) healthIndicator.getContributor("kstream"); + Health health = kafkaStreamsBinderHealthIndicator.health(); + while (waitFor(health.getStatus(), health.getDetails())) { + TimeUnit.SECONDS.sleep(2); + health = kafkaStreamsBinderHealthIndicator.health(); + } + assertThat(health.getStatus()).isEqualTo(expected); + } + + private ConfigurableApplicationContext singleStream(String applicationId) { + SpringApplication app = new SpringApplication(KStreamApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + return app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.input.destination=in", + "--spring.cloud.stream.bindings.output.destination=out", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde=" + + "org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde=" + + "org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId=" + + applicationId, + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString()); + } + + private ConfigurableApplicationContext multipleStream() { + System.setProperty("logging.level.org.apache.kafka", "OFF"); + SpringApplication app = new SpringApplication(AnotherKStreamApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + return app.run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.input.destination=in", + "--spring.cloud.stream.bindings.output.destination=out", + "--spring.cloud.stream.bindings.input2.destination=in2", + "--spring.cloud.stream.bindings.output2.destination=out2", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde=" + + "org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde=" + + "org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId=" + + "ApplicationHealthTest-xyz", + "--spring.cloud.stream.kafka.streams.bindings.input2.consumer.applicationId=" + + "ApplicationHealthTest2-xyz", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString()); + } + + @EnableBinding(KafkaStreamsProcessor.class) + @EnableAutoConfiguration + public static class KStreamApplication { + + @StreamListener("input") + @SendTo("output") + public KStream process(KStream input) { + return input.filter((key, product) -> { + if (product.getId() != 123) { + throw new IllegalArgumentException(); + } + return true; + }); + } + + } + + @EnableBinding({ KafkaStreamsProcessor.class, KafkaStreamsProcessorX.class }) + @EnableAutoConfiguration + public static class AnotherKStreamApplication { + + @StreamListener("input") + @SendTo("output") + public KStream process(KStream input) { + return input.filter((key, product) -> { + if (product.getId() != 123) { + throw new IllegalArgumentException(); + } + return true; + }); + } + + @StreamListener("input2") + @SendTo("output2") + public KStream process2(KStream input) { + return input.filter((key, product) -> { + if (product.getId() != 123) { + throw new IllegalArgumentException(); + } + return true; + }); + } + + @Bean + public StreamsBuilderFactoryBeanConfigurer customizer() { + return factoryBean -> { + factoryBean.setKafkaStreamsCustomizer(new KafkaStreamsCustomizer() { + @Override + public void customize(KafkaStreams kafkaStreams) { + kafkaStreams.setUncaughtExceptionHandler(exception -> + StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse.SHUTDOWN_CLIENT); + } + }); + }; + } + + } + + public interface KafkaStreamsProcessorX { + + @Input("input2") + KStream input(); + + @Output("output2") + KStream output(); + + } + + public static class Product { + + Integer id; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderMultipleInputTopicsTest.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderMultipleInputTopicsTest.java new file mode 100644 index 000000000..f857c3494 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderMultipleInputTopicsTest.java @@ -0,0 +1,208 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.CleanupConfig; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.messaging.handler.annotation.SendTo; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * This test case demonstrates a kafk-streams topology which consumes messages from + * multiple kafka topics(destinations). + * + * See + * {@link KafkaStreamsBinderMultipleInputTopicsTest#testKstreamWordCountWithStringInputAndPojoOuput} + * where the input topic names are specified as comma-separated String values for the + * property spring.cloud.stream.bindings.input.destination. + * + * @author Sarath Shyam + */ +public class KafkaStreamsBinderMultipleInputTopicsTest { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "counts"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + private static Consumer consumer; + + @BeforeClass + public static void setUp() throws Exception { + Map consumerProps = KafkaTestUtils.consumerProps("group", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "counts"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + } + + @Test + public void testKstreamWordCountWithStringInputAndPojoOuput() throws Exception { + SpringApplication app = new SpringApplication( + WordCountProcessorApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.input.destination=words1,words2", + "--spring.cloud.stream.bindings.output.destination=counts", + "--spring.cloud.stream.bindings.output.contentType=application/json", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.timeWindow.length=5000", + "--spring.cloud.stream.kafka.streams.timeWindow.advanceBy=0", + "--spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId" + + "=WordCountProcessorApplication-xyz", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString()); + try { + receiveAndValidate(); + } + finally { + context.close(); + } + } + + private void receiveAndValidate() + throws Exception { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("words1"); + template.sendDefault("foobar1"); + template.setDefaultTopic("words2"); + template.sendDefault("foobar2"); + // Sleep a bit so that both the messages are processed before reading from the + // output topic. + // Else assertions might fail arbitrarily. + Thread.sleep(5000); + ConsumerRecords received = KafkaTestUtils.getRecords(consumer); + List wordCounts = new ArrayList<>(2); + + received.records("counts") + .forEach((consumerRecord) -> wordCounts.add((consumerRecord.value()))); + System.out.println(wordCounts); + assertThat(wordCounts.contains("{\"word\":\"foobar1\",\"count\":1}")).isTrue(); + assertThat(wordCounts.contains("{\"word\":\"foobar2\",\"count\":1}")).isTrue(); + } + + @EnableBinding(KafkaStreamsProcessor.class) + @EnableAutoConfiguration + static class WordCountProcessorApplication { + + @StreamListener + @SendTo("output") + public KStream process( + @Input("input") KStream input) { + + input.map((k, v) -> { + System.out.println(k); + System.out.println(v); + return new KeyValue<>(k, v); + }); + return input + .flatMapValues( + value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .map((key, value) -> new KeyValue<>(value, value)) + .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) + .count(Materialized.as("WordCounts-tKWCWSIAP0")).toStream() + .map((key, value) -> new KeyValue<>(null, new WordCount(key, value))); + } + + @Bean + public CleanupConfig cleanupConfig() { + return new CleanupConfig(false, true); + } + + } + + static class WordCount { + + private String word; + + private long count; + + WordCount(String word, long count) { + this.word = word; + this.count = count; + } + + public String getWord() { + return word; + } + + public void setWord(String word) { + this.word = word; + } + + public long getCount() { + return count; + } + + public void setCount(long count) { + this.count = count; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java new file mode 100644 index 000000000..bda892780 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java @@ -0,0 +1,159 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration; + +import java.time.Duration; +import java.util.Map; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.serialization.LongDeserializer; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.serializer.JsonSerde; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.messaging.handler.annotation.SendTo; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Soby Chacko + * @author Gary Russell + */ +public class KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "counts-id"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + private static Consumer consumer; + + @BeforeClass + public static void setUp() throws Exception { + Map consumerProps = KafkaTestUtils.consumerProps("group-id", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put("value.deserializer", LongDeserializer.class); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "counts-id"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + } + + @Test + public void testKstreamBinderWithPojoInputAndStringOuput() throws Exception { + SpringApplication app = new SpringApplication(ProductCountApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.input.destination=foos", + "--spring.cloud.stream.bindings.output.destination=counts-id", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde=" + + "org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde=" + + "org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId=" + + "KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests-xyz", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString()); + try { + receiveAndValidateFoo(); + } + finally { + context.close(); + } + } + + private void receiveAndValidateFoo() { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("foos"); + template.sendDefault("{\"id\":\"123\"}"); + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, + "counts-id"); + + assertThat(cr.key()).isEqualTo(123); + assertThat(cr.value()).isEqualTo(1L); + } + + @EnableBinding(KafkaStreamsProcessor.class) + @EnableAutoConfiguration + public static class ProductCountApplication { + + @StreamListener("input") + @SendTo("output") + public KStream process(KStream input) { + return input.filter((key, product) -> product.getId() == 123) + .map((key, value) -> new KeyValue<>(value, value)) + .groupByKey(Grouped.with(new JsonSerde<>(Product.class), + new JsonSerde<>(Product.class))) + .windowedBy(TimeWindows.of(Duration.ofMillis(5000))) + .count(Materialized.as("id-count-store-x")).toStream() + .map((key, value) -> { + return new KeyValue<>(key.key().id, value); + }); + } + + } + + public static class Product { + + Integer id; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderTombstoneTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderTombstoneTests.java new file mode 100644 index 000000000..f8e118076 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderTombstoneTests.java @@ -0,0 +1,247 @@ +/* + * Copyright 2017-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Date; +import java.util.Map; +import java.util.Properties; +import java.util.function.Function; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.integration.test.util.TestUtils; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.core.CleanupConfig; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Marius Bogoevici + * @author Soby Chacko + * @author Gary Russell + */ +public class KafkaStreamsBinderTombstoneTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "counts-1"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + private static Consumer consumer; + + @BeforeClass + public static void setUp() { + Map consumerProps = KafkaTestUtils.consumerProps("group", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer, "counts-1"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + } + + @Test + public void testSendToTombstone() + throws Exception { + SpringApplication app = new SpringApplication( + WordCountProcessorApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.process-in-0.destination=words-1", + "--spring.cloud.stream.bindings.process-out-0.destination=counts-1", + "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.application-id=testKstreamWordCountWithInputBindingLevelApplicationId", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.bindings.process-out-0.producer.valueSerde=org.springframework.kafka.support.serializer.JsonSerde", + "--spring.cloud.stream.bindings.process-in-0.consumer.concurrency=2", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString())) { + receiveAndValidate("words-1", "counts-1"); + // Assertions on StreamBuilderFactoryBean + StreamsBuilderFactoryBean streamsBuilderFactoryBean = context + .getBean("&stream-builder-process", StreamsBuilderFactoryBean.class); + KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams(); + assertThat(kafkaStreams).isNotNull(); + // Ensure that concurrency settings are mapped to number of stream task + // threads in Kafka Streams. + final Properties streamsConfiguration = streamsBuilderFactoryBean.getStreamsConfiguration(); + final Integer concurrency = (Integer) streamsConfiguration + .get(StreamsConfig.NUM_STREAM_THREADS_CONFIG); + assertThat(concurrency).isEqualTo(2); + + sendTombStoneRecordsAndVerifyGracefulHandling(); + + CleanupConfig cleanup = TestUtils.getPropertyValue(streamsBuilderFactoryBean, + "cleanupConfig", CleanupConfig.class); + assertThat(cleanup.cleanupOnStart()).isTrue(); + assertThat(cleanup.cleanupOnStop()).isFalse(); + } + } + + private void receiveAndValidate(String in, String out) { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic(in); + template.sendDefault("foobar"); + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, + out); + assertThat(cr.value().contains("\"word\":\"foobar\",\"count\":1")).isTrue(); + } + finally { + pf.destroy(); + } + } + + private void sendTombStoneRecordsAndVerifyGracefulHandling() throws Exception { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("words-1"); + template.sendDefault(null); + ConsumerRecords received = consumer + .poll(Duration.ofMillis(5000)); + // By asserting that the received record is empty, we are ensuring that the + // tombstone record + // was handled by the binder gracefully. + assertThat(received.isEmpty()).isTrue(); + } + finally { + pf.destroy(); + } + } + + @EnableAutoConfiguration + static class WordCountProcessorApplication { + + @Bean + public Function, KStream> process() { + + return input -> input + .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .map((key, value) -> new KeyValue<>(value, value)) + .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) + .windowedBy(TimeWindows.of(Duration.ofMillis(5000))) + .count(Materialized.as("foo-WordCounts")) + .toStream() + .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, + new Date(key.window().start()), new Date(key.window().end())))); + } + + @Bean + public CleanupConfig cleanupConfig() { + return new CleanupConfig(true, false); + } + + } + + static class WordCount { + + private String word; + + private long count; + + private Date start; + + private Date end; + + WordCount(String word, long count, Date start, Date end) { + this.word = word; + this.count = count; + this.start = start; + this.end = end; + } + + public String getWord() { + return word; + } + + public void setWord(String word) { + this.word = word; + } + + public long getCount() { + return count; + } + + public void setCount(long count) { + this.count = count; + } + + public Date getStart() { + return start; + } + + public void setStart(Date start) { + this.start = start; + } + + public Date getEnd() { + return end; + } + + public void setEnd(Date end) { + this.end = end; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsNativeEncodingDecodingTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsNativeEncodingDecodingTests.java new file mode 100644 index 000000000..4bb5abb46 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsNativeEncodingDecodingTests.java @@ -0,0 +1,186 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Map; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.messaging.handler.annotation.SendTo; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.StopWatch; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +/** + * @author Soby Chacko + */ +@RunWith(SpringRunner.class) +@ContextConfiguration +@DirtiesContext +public abstract class KafkaStreamsNativeEncodingDecodingTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "decode-counts", "decode-counts-1"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + @SpyBean + org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsMessageConversionDelegate conversionDelegate; + + private static Consumer consumer; + + @BeforeClass + public static void setUp() { + System.setProperty("spring.cloud.stream.kafka.streams.binder.brokers", + embeddedKafka.getBrokersAsString()); + System.setProperty("server.port", "0"); + System.setProperty("spring.jmx.enabled", "false"); + + Map consumerProps = KafkaTestUtils.consumerProps("group", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer, "decode-counts", "decode-counts-1"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + System.clearProperty("spring.cloud.stream.kafka.streams.binder.brokers"); + System.clearProperty("server.port"); + System.clearProperty("spring.jmx.enabled"); + } + + @SpringBootTest(properties = { + "spring.cloud.stream.bindings.input.destination=decode-words-1", + "spring.cloud.stream.bindings.output.destination=decode-counts-1", + "spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId" + + "=NativeEncodingDecodingEnabledTests-abc" }, webEnvironment = SpringBootTest.WebEnvironment.NONE) + public static class NativeEncodingDecodingEnabledTests + extends KafkaStreamsNativeEncodingDecodingTests { + + @Test + public void test() throws Exception { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("decode-words-1"); + template.sendDefault("foobar"); + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, + "decode-counts-1"); + assertThat(cr.value().equals("Count for foobar : 1")).isTrue(); + + verify(conversionDelegate, never()).serializeOnOutbound(any(KStream.class)); + verify(conversionDelegate, never()).deserializeOnInbound(any(Class.class), + any(KStream.class)); + } + + } + + @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { + "spring.cloud.stream.bindings.input.destination=decode-words", + "spring.cloud.stream.bindings.output.destination=decode-counts", + "spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", + "spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", + "spring.cloud.stream.kafka.streams.bindings.input3.consumer.applicationId" + + "=hello-NativeEncodingDecodingEnabledTests-xyz" }) + public static class NativeEncodingDecodingDisabledTests + extends KafkaStreamsNativeEncodingDecodingTests { + + @Test + public void test() { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("decode-words"); + template.sendDefault("foobar"); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + System.out.println("Starting: "); + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, + "decode-counts"); + stopWatch.stop(); + System.out.println("Total time: " + stopWatch.getTotalTimeSeconds()); + assertThat(cr.value().equals("Count for foobar : 1")).isTrue(); + + verify(conversionDelegate).serializeOnOutbound(any(KStream.class)); + verify(conversionDelegate).deserializeOnInbound(any(Class.class), + any(KStream.class)); + } + + } + + @EnableBinding(KafkaStreamsProcessor.class) + @EnableAutoConfiguration + public static class WordCountProcessorApplication { + + @StreamListener("input") + @SendTo("output") + public KStream process(KStream input) { + + return input + .flatMapValues( + value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .map((key, value) -> new KeyValue<>(value, value)) + .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) + .windowedBy(TimeWindows.of(Duration.ofSeconds(5))).count(Materialized.as("foo-WordCounts-x")) + .toStream().map((key, value) -> new KeyValue<>(null, + "Count for " + key.key() + " : " + value)); + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsStateStoreIntegrationTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsStateStoreIntegrationTests.java new file mode 100644 index 000000000..2cc15053e --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsStateStoreIntegrationTests.java @@ -0,0 +1,339 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration; + +import java.time.Duration; +import java.util.Map; + +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.processor.Processor; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.state.StoreBuilder; +import org.apache.kafka.streams.state.Stores; +import org.apache.kafka.streams.state.WindowStore; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsStateStore; +import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsStateStoreProperties; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; + +import static junit.framework.TestCase.fail; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Lei Chen + * @author Soby Chacko + */ +public class KafkaStreamsStateStoreIntegrationTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "counts-id"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + @Test + public void testKstreamStateStore() throws Exception { + SpringApplication app = new SpringApplication(ProductCountApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.input.destination=foobar", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId" + + "=KafkaStreamsStateStoreIntegrationTests-abc", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString()); + try { + Thread.sleep(2000); + receiveAndValidateFoo(context, ProductCountApplication.class); + } + catch (Exception e) { + throw e; + } + finally { + context.close(); + } + } + + @Test + public void testKstreamStateStoreBuilderBeansDefinedInApplication() throws Exception { + SpringApplication app = new SpringApplication(StateStoreBeanApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.input3.destination=foobar", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.bindings.input3.consumer.applicationId" + + "=KafkaStreamsStateStoreIntegrationTests-xyzabc-123", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString()); + try { + Thread.sleep(2000); + receiveAndValidateFoo(context, StateStoreBeanApplication.class); + } + catch (Exception e) { + throw e; + } + finally { + context.close(); + } + } + + + @Test + public void testSameStateStoreIsCreatedOnlyOnceWhenMultipleInputBindingsArePresent() throws Exception { + SpringApplication app = new SpringApplication(ProductCountApplicationWithMultipleInputBindings.class); + app.setWebApplicationType(WebApplicationType.NONE); + ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.input1.destination=foobar", + "--spring.cloud.stream.bindings.input2.destination=hello-foobar", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.bindings.input1.consumer.applicationId" + + "=KafkaStreamsStateStoreIntegrationTests-abc", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString()); + try { + Thread.sleep(2000); + // We are not particularly interested in querying the state store here, as that is verified by the other test + // in this class. This test verifies that the same store is not attempted to be created by multiple input bindings. + // Normally, that will cause an exception to be thrown. However by not getting any exceptions, we are verifying + // that the binder is handling it appropriately. + //For more info, see this issue: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/551 + } + catch (Exception e) { + throw e; + } + finally { + context.close(); + } + } + + private void receiveAndValidateFoo(ConfigurableApplicationContext context, Class clazz) + throws Exception { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("foobar"); + template.sendDefault("{\"id\":\"123\"}"); + Thread.sleep(1000); + + // assertions + if (clazz.isAssignableFrom(ProductCountApplication.class)) { + ProductCountApplication productCount = context + .getBean(ProductCountApplication.class); + WindowStore state = productCount.state; + assertThat(state != null).isTrue(); + assertThat(state.name()).isEqualTo("mystate"); + assertThat(state.persistent()).isTrue(); + assertThat(productCount.processed).isTrue(); + } + else if (clazz.isAssignableFrom(StateStoreBeanApplication.class)) { + StateStoreBeanApplication productCount = context + .getBean(StateStoreBeanApplication.class); + WindowStore state = productCount.state; + assertThat(state != null).isTrue(); + assertThat(state.name()).isEqualTo("mystate"); + assertThat(state.persistent()).isTrue(); + assertThat(productCount.processed).isTrue(); + } + else { + fail("Expected assertiond did not happen"); + } + + } + + @EnableBinding(KafkaStreamsProcessorX.class) + @EnableAutoConfiguration + public static class ProductCountApplication { + + WindowStore state; + + boolean processed; + + @StreamListener("input") + @KafkaStreamsStateStore(name = "mystate", type = KafkaStreamsStateStoreProperties.StoreType.WINDOW, lengthMs = 300000, retentionMs = 300000) + @SuppressWarnings({ "deprecation", "unchecked" }) + public void process(KStream input) { + + input.process(() -> new Processor() { + + @Override + public void init(ProcessorContext processorContext) { + state = (WindowStore) processorContext.getStateStore("mystate"); + } + + @Override + public void process(Object s, Product product) { + processed = true; + } + + @Override + public void close() { + if (state != null) { + state.close(); + } + } + }, "mystate"); + } + } + + @EnableBinding(KafkaStreamsProcessorZ.class) + @EnableAutoConfiguration + public static class StateStoreBeanApplication { + + WindowStore state; + + boolean processed; + + @StreamListener("input3") + @SuppressWarnings({"unchecked" }) + public void process(KStream input) { + + input.process(() -> new Processor() { + + @Override + public void init(ProcessorContext processorContext) { + state = (WindowStore) processorContext.getStateStore("mystate"); + } + + @Override + public void process(Object s, Product product) { + processed = true; + } + + @Override + public void close() { + if (state != null) { + state.close(); + } + } + }, "mystate"); + } + + @Bean + public StoreBuilder mystore() { + return Stores.windowStoreBuilder( + Stores.persistentWindowStore("mystate", + Duration.ofMillis(3), Duration.ofMillis(3), false), Serdes.String(), + Serdes.String()); + } + } + + + @EnableBinding(KafkaStreamsProcessorY.class) + @EnableAutoConfiguration + public static class ProductCountApplicationWithMultipleInputBindings { + + WindowStore state; + + boolean processed; + + @StreamListener + @KafkaStreamsStateStore(name = "mystate", type = KafkaStreamsStateStoreProperties.StoreType.WINDOW, lengthMs = 300000, retentionMs = 300000) + @SuppressWarnings({ "deprecation", "unchecked" }) + public void process(@Input("input1")KStream input, @Input("input2")KStream input2) { + + input.process(() -> new Processor() { + + @Override + public void init(ProcessorContext processorContext) { + state = (WindowStore) processorContext.getStateStore("mystate"); + } + + @Override + public void process(Object s, Product product) { + processed = true; + } + + @Override + public void close() { + if (state != null) { + state.close(); + } + } + }, "mystate"); + + //simple use of input2, we are not using input2 for anything other than triggering some test behavior. + input2.foreach((key, value) -> { }); + } + } + + public static class Product { + + Integer id; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + } + + interface KafkaStreamsProcessorX { + + @Input("input") + KStream input(); + } + + interface KafkaStreamsProcessorY { + + @Input("input1") + KStream input1(); + + @Input("input2") + KStream input2(); + } + + interface KafkaStreamsProcessorZ { + + @Input("input3") + KStream input3(); + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkastreamsBinderPojoInputStringOutputIntegrationTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkastreamsBinderPojoInputStringOutputIntegrationTests.java new file mode 100644 index 000000000..2ff80f252 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkastreamsBinderPojoInputStringOutputIntegrationTests.java @@ -0,0 +1,165 @@ +/* + * Copyright 2017-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration; + +import java.time.Duration; +import java.util.Map; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.integration.test.util.TestUtils; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.core.CleanupConfig; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.serializer.JsonSerde; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.messaging.handler.annotation.SendTo; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Marius Bogoevici + * @author Soby Chacko + * @author Gary Russell + */ +public class KafkastreamsBinderPojoInputStringOutputIntegrationTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "counts-id"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + private static Consumer consumer; + + @BeforeClass + public static void setUp() throws Exception { + Map consumerProps = KafkaTestUtils.consumerProps("group-id", + "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "counts-id"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + } + + @Test + public void testKstreamBinderWithPojoInputAndStringOuput() throws Exception { + SpringApplication app = new SpringApplication(ProductCountApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.input.destination=foos", + "--spring.cloud.stream.bindings.output.destination=counts-id", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + + "=org.apache.kafka.common.serialization.Serdes$StringSerde", + "--spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId=ProductCountApplication-xyz", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString()); + try { + receiveAndValidateFoo(); + // Assertions on StreamBuilderFactoryBean + StreamsBuilderFactoryBean streamsBuilderFactoryBean = context + .getBean("&stream-builder-ProductCountApplication-process", StreamsBuilderFactoryBean.class); + CleanupConfig cleanup = TestUtils.getPropertyValue(streamsBuilderFactoryBean, + "cleanupConfig", CleanupConfig.class); + assertThat(cleanup.cleanupOnStart()).isFalse(); + assertThat(cleanup.cleanupOnStop()).isFalse(); + } + finally { + context.close(); + } + } + + private void receiveAndValidateFoo() { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("foos"); + template.sendDefault("{\"id\":\"123\"}"); + ConsumerRecord cr = KafkaTestUtils.getSingleRecord(consumer, + "counts-id"); + assertThat(cr.value().contains("Count for product with ID 123: 1")).isTrue(); + } + + @EnableBinding(KafkaStreamsProcessor.class) + @EnableAutoConfiguration + public static class ProductCountApplication { + + @StreamListener("input") + @SendTo("output") + public KStream process(KStream input) { + + return input.filter((key, product) -> product.getId() == 123) + .map((key, value) -> new KeyValue<>(value, value)) + .groupByKey(Grouped.with(new JsonSerde<>(Product.class), + new JsonSerde<>(Product.class))) + .windowedBy(TimeWindows.of(Duration.ofMillis(5000))) + .count(Materialized.as("id-count-store")).toStream() + .map((key, value) -> new KeyValue<>(key.key().id, + "Count for product with ID 123: " + value)); + } + + } + + static class Product { + + Integer id; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/MultiProcessorsWithSameNameAndBindingTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/MultiProcessorsWithSameNameAndBindingTests.java new file mode 100644 index 000000000..699cee4a7 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/MultiProcessorsWithSameNameAndBindingTests.java @@ -0,0 +1,95 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration; + +import org.apache.kafka.streams.kstream.KStream; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.stereotype.Component; + +import static org.assertj.core.api.Assertions.assertThat; + +public class MultiProcessorsWithSameNameAndBindingTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "counts"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + @Test + public void testBinderStartsSuccessfullyWhenTwoProcessorsWithSameNamesAndBindingsPresent() { + SpringApplication app = new SpringApplication( + MultiProcessorsWithSameNameAndBindingTests.WordCountProcessorApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext context = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.input.destination=words", + "--spring.cloud.stream.bindings.input-1.destination=words", + "--spring.cloud.stream.bindings.output.destination=counts", + "--spring.cloud.stream.bindings.output.contentType=application/json", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString())) { + StreamsBuilderFactoryBean streamsBuilderFactoryBean1 = context + .getBean("&stream-builder-Foo-process", StreamsBuilderFactoryBean.class); + assertThat(streamsBuilderFactoryBean1).isNotNull(); + StreamsBuilderFactoryBean streamsBuilderFactoryBean2 = context + .getBean("&stream-builder-Bar-process", StreamsBuilderFactoryBean.class); + assertThat(streamsBuilderFactoryBean2).isNotNull(); + } + } + + @EnableBinding(KafkaStreamsProcessorX.class) + @EnableAutoConfiguration + static class WordCountProcessorApplication { + + @Component + static class Foo { + @StreamListener + public void process(@Input("input-1") KStream input) { + } + } + + //Second class with a stub processor that has the same name as above ("process") + @Component + static class Bar { + @StreamListener + public void process(@Input("input-1") KStream input) { + } + } + } + + interface KafkaStreamsProcessorX { + + @Input("input-1") + KStream input1(); + + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/PerRecordAvroContentTypeTests.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/PerRecordAvroContentTypeTests.java new file mode 100644 index 000000000..59eee2591 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/PerRecordAvroContentTypeTests.java @@ -0,0 +1,184 @@ +/* + * Copyright 2017-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration; + +import java.io.IOException; +import java.util.Map; +import java.util.Random; +import java.util.UUID; + +import com.example.Sensor; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.KStream; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.function.context.converter.avro.AvroSchemaMessageConverter; +import org.springframework.cloud.function.context.converter.avro.AvroSchemaServiceManagerImpl; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsProcessor; +import org.springframework.cloud.stream.binder.kafka.streams.integration.utils.TestAvroSerializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.messaging.Message; +import org.springframework.messaging.converter.MessageConverter; +import org.springframework.messaging.handler.annotation.SendTo; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.util.MimeTypeUtils; + +import static org.assertj.core.api.Assertions.assertThat; + + +/** + * @author Soby Chacko + */ +public class PerRecordAvroContentTypeTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, + "received-sensors"); + + private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule + .getEmbeddedKafka(); + + private static Consumer consumer; + + @BeforeClass + public static void setUp() throws Exception { + Map consumerProps = KafkaTestUtils.consumerProps("avro-ct-test", + "false", embeddedKafka); + + // Receive the data as byte[] + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + ByteArrayDeserializer.class); + + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>( + consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "received-sensors"); + } + + @AfterClass + public static void tearDown() { + consumer.close(); + } + + @Test + public void testPerRecordAvroConentTypeAndVerifySerialization() throws Exception { + SpringApplication app = new SpringApplication(SensorCountAvroApplication.class); + app.setWebApplicationType(WebApplicationType.NONE); + + try (ConfigurableApplicationContext ignored = app.run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.input.consumer.useNativeDecoding=false", + "--spring.cloud.stream.bindings.output.producer.useNativeEncoding=false", + "--spring.cloud.stream.bindings.input.destination=sensors", + "--spring.cloud.stream.bindings.output.destination=received-sensors", + "--spring.cloud.stream.bindings.output.contentType=application/avro", + "--spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=per-record-avro-contentType-test", + "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000", + "--spring.cloud.stream.kafka.streams.binder.brokers=" + + embeddedKafka.getBrokersAsString())) { + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + // Use a custom avro test serializer + senderProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + TestAvroSerializer.class); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>( + senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + + Random random = new Random(); + Sensor sensor = new Sensor(); + sensor.setId(UUID.randomUUID().toString() + "-v1"); + sensor.setAcceleration(random.nextFloat() * 10); + sensor.setVelocity(random.nextFloat() * 100); + sensor.setTemperature(random.nextFloat() * 50); + // Send with avro content type set. + Message message = MessageBuilder.withPayload(sensor) + .setHeader("contentType", "application/avro").build(); + template.setDefaultTopic("sensors"); + template.send(message); + + // Serialized byte[] ^^ is received by the binding process and deserialzed + // it using avro converter. + // Then finally, the data will be output to a return topic as byte[] + // (using the same avro converter). + + // Receive the byte[] from return topic + ConsumerRecord cr = KafkaTestUtils + .getSingleRecord(consumer, "received-sensors"); + final byte[] value = cr.value(); + + // Convert the byte[] received back to avro object and verify that it is + // the same as the one we sent ^^. + AvroSchemaMessageConverter avroSchemaMessageConverter = new AvroSchemaMessageConverter(new AvroSchemaServiceManagerImpl()); + + Message receivedMessage = MessageBuilder.withPayload(value) + .setHeader("contentType", + MimeTypeUtils.parseMimeType("application/avro")) + .build(); + Sensor messageConverted = (Sensor) avroSchemaMessageConverter + .fromMessage(receivedMessage, Sensor.class); + assertThat(messageConverted).isEqualTo(sensor); + } + finally { + pf.destroy(); + } + } + } + + @EnableBinding(KafkaStreamsProcessor.class) + @EnableAutoConfiguration + static class SensorCountAvroApplication { + + @StreamListener + @SendTo("output") + public KStream process(@Input("input") KStream input) { + // return the same Sensor object unchanged so that we can do test + // verifications + return input.map(KeyValue::new); + } + + @Bean + public MessageConverter sensorMessageConverter() throws IOException { + return new AvroSchemaMessageConverter(new AvroSchemaServiceManagerImpl()); + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/utils/TestAvroSerializer.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/utils/TestAvroSerializer.java new file mode 100644 index 000000000..761636ea9 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/utils/TestAvroSerializer.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.integration.utils; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.kafka.common.serialization.Serializer; + +import org.springframework.cloud.function.context.converter.avro.AvroSchemaMessageConverter; +import org.springframework.cloud.function.context.converter.avro.AvroSchemaServiceManagerImpl; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.MessageBuilder; + +/** + * Custom avro serializer intended to be used for testing only. + * + * @param Target type to serialize + * @author Soby Chacko + */ +public class TestAvroSerializer implements Serializer { + + public TestAvroSerializer() { + } + + @Override + public void configure(Map configs, boolean isKey) { + + } + + @Override + public byte[] serialize(String topic, S data) { + AvroSchemaMessageConverter avroSchemaMessageConverter = new AvroSchemaMessageConverter(new AvroSchemaServiceManagerImpl()); + Message message = MessageBuilder.withPayload(data).build(); + Map headers = new HashMap<>(message.getHeaders()); + headers.put(MessageHeaders.CONTENT_TYPE, "application/avro"); + MessageHeaders messageHeaders = new MessageHeaders(headers); + final Object payload = avroSchemaMessageConverter + .toMessage(message.getPayload(), messageHeaders).getPayload(); + return (byte[]) payload; + } + + @Override + public void close() { + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CollectionSerdeTest.java b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CollectionSerdeTest.java new file mode 100644 index 000000000..0b060d7a5 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CollectionSerdeTest.java @@ -0,0 +1,89 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.streams.serde; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * + * @author Soby Chacko + */ +public class CollectionSerdeTest { + + @Test + public void testCollectionsSerde() { + + Foo foo1 = new Foo(); + foo1.setData("data-1"); + foo1.setNum(1); + + Foo foo2 = new Foo(); + foo2.setData("data-2"); + foo2.setNum(2); + + List foos = new ArrayList<>(); + foos.add(foo1); + foos.add(foo2); + + CollectionSerde collectionSerde = new CollectionSerde<>(Foo.class, ArrayList.class); + byte[] serialized = collectionSerde.serializer().serialize("", foos); + + Collection deserialized = collectionSerde.deserializer().deserialize("", serialized); + + Iterator iterator = deserialized.iterator(); + Foo foo1Retrieved = iterator.next(); + assertThat(foo1Retrieved.getData()).isEqualTo("data-1"); + assertThat(foo1Retrieved.getNum()).isEqualTo(1); + + Foo foo2Retrieved = iterator.next(); + assertThat(foo2Retrieved.getData()).isEqualTo("data-2"); + assertThat(foo2Retrieved.getNum()).isEqualTo(2); + + } + + static class Foo { + + private int num; + private String data; + + Foo() { + } + + public int getNum() { + return num; + } + + public void setNum(int num) { + this.num = num; + } + + public String getData() { + return data; + } + + public void setData(String data) { + this.data = data; + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/resources/avro/sensor.avsc b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/resources/avro/sensor.avsc new file mode 100644 index 000000000..c0e060d3d --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/resources/avro/sensor.avsc @@ -0,0 +1,11 @@ +{ + "namespace" : "com.example", + "type" : "record", + "name" : "Sensor", + "fields" : [ + {"name":"id","type":"string"}, + {"name":"temperature", "type":"float", "default":0.0}, + {"name":"acceleration", "type":"float","default":0.0}, + {"name":"velocity","type":"float","default":0.0} + ] +} diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/resources/logback.xml b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/resources/logback.xml new file mode 100644 index 000000000..414797f87 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/resources/logback.xml @@ -0,0 +1,16 @@ + + + + %d{ISO8601} %5p %t %c{2}:%L - %m%n + + + + + + + + + + + + diff --git a/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/resources/org/springframework/cloud/stream/binder/kstream/integTest-1.properties b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/resources/org/springframework/cloud/stream/binder/kstream/integTest-1.properties new file mode 100644 index 000000000..6d983a0b1 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka-streams/src/test/resources/org/springframework/cloud/stream/binder/kstream/integTest-1.properties @@ -0,0 +1,6 @@ +spring.cloud.stream.bindings.input.destination=DeserializationErrorHandlerByKafkaTests-In +spring.cloud.stream.bindings.output.destination=DeserializationErrorHandlerByKafkaTests-Out +spring.cloud.stream.bindings.output.contentType=application/json +spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000 +spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde=org.apache.kafka.common.serialization.Serdes$StringSerde +spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde=org.apache.kafka.common.serialization.Serdes$StringSerde diff --git a/k-binder/spring-cloud-stream-binder-kafka/.jdk8 b/k-binder/spring-cloud-stream-binder-kafka/.jdk8 new file mode 100644 index 000000000..e69de29bb diff --git a/k-binder/spring-cloud-stream-binder-kafka/.settings/org.eclipse.jdt.ui.prefs b/k-binder/spring-cloud-stream-binder-kafka/.settings/org.eclipse.jdt.ui.prefs new file mode 100644 index 000000000..9b721a32a --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/.settings/org.eclipse.jdt.ui.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.ui.ignorelowercasenames=true +org.eclipse.jdt.ui.importorder=java;javax;com;io.micrometer;org;org.springframework;ch.qos;\#; +org.eclipse.jdt.ui.ondemandthreshold=99 +org.eclipse.jdt.ui.staticondemandthreshold=99 diff --git a/k-binder/spring-cloud-stream-binder-kafka/pom.xml b/k-binder/spring-cloud-stream-binder-kafka/pom.xml new file mode 100644 index 000000000..92187b719 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/pom.xml @@ -0,0 +1,85 @@ + + + 4.0.0 + + spring-cloud-stream-binder-kafka + jar + spring-cloud-stream-binder-kafka + Kafka binder implementation + + + org.springframework.cloud + spring-cloud-stream-binder-kafka-parent + 3.2.3-SNAPSHOT + + + + + org.springframework.cloud + spring-cloud-stream-binder-kafka-core + + + org.springframework.boot + spring-boot-starter-actuator + true + + + org.springframework.boot + spring-boot-configuration-processor + true + + + org.springframework.cloud + spring-cloud-stream + + + org.springframework.boot + spring-boot-autoconfigure + true + + + org.apache.kafka + kafka-clients + + + org.springframework.kafka + spring-kafka + + + org.springframework.boot + spring-boot-test + test + + + org.springframework.kafka + spring-kafka-test + test + + + org.springframework.cloud + spring-cloud-stream-binder-test + test + + + + org.apache.kafka + kafka-clients + test + + + org.apache.kafka + kafka_2.13 + + + org.apache.kafka + kafka_2.13 + test + + + org.awaitility + awaitility + test + + + + diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/BinderHeaderMapper.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/BinderHeaderMapper.java new file mode 100644 index 000000000..9e5dd435c --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/BinderHeaderMapper.java @@ -0,0 +1,508 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdNodeBasedDeserializer; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.databind.type.TypeFactory; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.header.internals.RecordHeader; + +import org.springframework.cloud.stream.binder.BinderHeaders; +import org.springframework.integration.IntegrationMessageHeaderAccessor; +import org.springframework.kafka.support.AbstractKafkaHeaderMapper; +import org.springframework.kafka.support.JacksonUtils; +import org.springframework.lang.Nullable; +import org.springframework.messaging.MessageHeaders; +import org.springframework.util.Assert; +import org.springframework.util.ClassUtils; +import org.springframework.util.MimeType; + +/** + * Custom header mapper for Apache Kafka. This is identical to the {@link org.springframework.kafka.support.DefaultKafkaHeaderMapper} + * from spring Kafka. This is provided for addressing some interoperability issues between Spring Cloud Stream 3.0.x + * and 2.x apps, where mime types passed as regular {@link MimeType} in the header are not de-serialized properly. + * It also suppresses certain internal headers that should never be propagated on output. + * + * Most headers in {@link org.springframework.kafka.support.KafkaHeaders} are not mapped onto outbound messages. + * The exceptions are correlation and reply headers for request/reply + * messaging. + * Header types are added to a special header {@link #JSON_TYPES}. + * + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + * + * @since 3.0.0 + * + */ +public class BinderHeaderMapper extends AbstractKafkaHeaderMapper { + + private static final String NEGATE = "!"; + + private static final String NEVER_ID = NEGATE + MessageHeaders.ID; + + private static final String NEVER_TIMESTAMP = NEGATE + MessageHeaders.TIMESTAMP; + + private static final String NEVER_DELIVERY_ATTEMPT = NEGATE + IntegrationMessageHeaderAccessor.DELIVERY_ATTEMPT; + + private static final String NEVER_NATIVE_HEADERS_PRESENT = NEGATE + BinderHeaders.NATIVE_HEADERS_PRESENT; + + private static final String JAVA_LANG_STRING = "java.lang.String"; + + private static final List DEFAULT_TRUSTED_PACKAGES = + Arrays.asList( + "java.lang", + "java.net", + "java.util", + "org.springframework.util" + ); + + private static final List DEFAULT_TO_STRING_CLASSES = + Arrays.asList( + "org.springframework.util.MimeType", + "org.springframework.http.MediaType" + ); + + /** + * Header name for java types of other headers. + */ + public static final String JSON_TYPES = "spring_json_header_types"; + + private final ObjectMapper objectMapper; + + private final Set trustedPackages = new LinkedHashSet<>(DEFAULT_TRUSTED_PACKAGES); + + private final Set toStringClasses = new LinkedHashSet<>(DEFAULT_TO_STRING_CLASSES); + + private boolean encodeStrings; + + /** + * Construct an instance with the default object mapper and default header patterns + * for outbound headers; all inbound headers are mapped. The default pattern list is + * {@code "!id", "!timestamp" and "*"}. In addition, most of the headers in + * {@link org.springframework.kafka.support.KafkaHeaders} are never mapped as headers since they represent data in + * consumer/producer records. + * @see #BinderHeaderMapper(ObjectMapper) + */ + public BinderHeaderMapper() { + this(JacksonUtils.enhancedObjectMapper()); + } + + /** + * Construct an instance with the provided object mapper and default header patterns + * for outbound headers; all inbound headers are mapped. The patterns are applied in + * order, stopping on the first match (positive or negative). Patterns are negated by + * preceding them with "!". The default pattern list is + * {@code "!id", "!timestamp" and "*"}. In addition, most of the headers in + * {@link org.springframework.kafka.support.KafkaHeaders} are never mapped as headers since they represent data in + * consumer/producer records. + * @param objectMapper the object mapper. + * @see org.springframework.util.PatternMatchUtils#simpleMatch(String, String) + */ + public BinderHeaderMapper(ObjectMapper objectMapper) { + this(objectMapper, + NEVER_ID, + NEVER_TIMESTAMP, + NEVER_DELIVERY_ATTEMPT, + NEVER_NATIVE_HEADERS_PRESENT, + "*"); + } + + /** + * Construct an instance with a default object mapper and the provided header patterns + * for outbound headers; all inbound headers are mapped. The patterns are applied in + * order, stopping on the first match (positive or negative). Patterns are negated by + * preceding them with "!". The patterns will replace the default patterns; you + * generally should not map the {@code "id" and "timestamp"} headers. Note: + * most of the headers in {@link org.springframework.kafka.support.KafkaHeaders} are ever mapped as headers since they + * represent data in consumer/producer records. + * @param patterns the patterns. + * @see org.springframework.util.PatternMatchUtils#simpleMatch(String, String) + */ + public BinderHeaderMapper(String... patterns) { + this(new ObjectMapper(), patterns); + } + + /** + * Construct an instance with the provided object mapper and the provided header + * patterns for outbound headers; all inbound headers are mapped. The patterns are + * applied in order, stopping on the first match (positive or negative). Patterns are + * negated by preceding them with "!". The patterns will replace the default patterns; + * you generally should not map the {@code "id" and "timestamp"} headers. Note: most + * of the headers in {@link org.springframework.kafka.support.KafkaHeaders} are never mapped as headers since they + * represent data in consumer/producer records. + * @param objectMapper the object mapper. + * @param patterns the patterns. + * @see org.springframework.util.PatternMatchUtils#simpleMatch(String, String) + */ + public BinderHeaderMapper(ObjectMapper objectMapper, String... patterns) { + super(patterns); + Assert.notNull(objectMapper, "'objectMapper' must not be null"); + Assert.noNullElements(patterns, "'patterns' must not have null elements"); + this.objectMapper = objectMapper; + this.objectMapper + .registerModule(new SimpleModule().addDeserializer(MimeType.class, new MimeTypeJsonDeserializer())); + } + + /** + * Return the object mapper. + * @return the mapper. + */ + protected ObjectMapper getObjectMapper() { + return this.objectMapper; + } + + /** + * Provide direct access to the trusted packages set for subclasses. + * @return the trusted packages. + * @since 2.2 + */ + protected Set getTrustedPackages() { + return this.trustedPackages; + } + + /** + * Provide direct access to the toString() classes by subclasses. + * @return the toString() classes. + * @since 2.2 + */ + protected Set getToStringClasses() { + return this.toStringClasses; + } + + protected boolean isEncodeStrings() { + return this.encodeStrings; + } + + /** + * Set to true to encode String-valued headers as JSON ("..."), by default just the + * raw String value is converted to a byte array using the configured charset. Set to + * true if a consumer of the outbound record is using Spring for Apache Kafka version + * less than 2.3 + * @param encodeStrings true to encode (default false). + * @since 2.3 + */ + public void setEncodeStrings(boolean encodeStrings) { + this.encodeStrings = encodeStrings; + } + + /** + * Add packages to the trusted packages list (default {@code java.util, java.lang}) used + * when constructing objects from JSON. + * If any of the supplied packages is {@code "*"}, all packages are trusted. + * If a class for a non-trusted package is encountered, the header is returned to the + * application with value of type {@link NonTrustedHeaderType}. + * @param packagesToTrust the packages to trust. + */ + public void addTrustedPackages(String... packagesToTrust) { + if (packagesToTrust != null) { + for (String whiteList : packagesToTrust) { + if ("*".equals(whiteList)) { + this.trustedPackages.clear(); + break; + } + else { + this.trustedPackages.add(whiteList); + } + } + } + } + + /** + * Add class names that the outbound mapper should perform toString() operations on + * before mapping. + * @param classNames the class names. + * @since 2.2 + */ + public void addToStringClasses(String... classNames) { + this.toStringClasses.addAll(Arrays.asList(classNames)); + } + + @Override + public void fromHeaders(MessageHeaders headers, Headers target) { + final Map jsonHeaders = new HashMap<>(); + final ObjectMapper headerObjectMapper = getObjectMapper(); + headers.forEach((key, rawValue) -> { + if (matches(key, rawValue)) { + Object valueToAdd = headerValueToAddOut(key, rawValue); + if (valueToAdd instanceof byte[]) { + target.add(new RecordHeader(key, (byte[]) valueToAdd)); + } + else { + try { + String className = valueToAdd.getClass().getName(); + if (this.toStringClasses.contains(className)) { + valueToAdd = valueToAdd.toString(); + className = JAVA_LANG_STRING; + } + if (!this.encodeStrings + && !MimeType.class.isAssignableFrom(rawValue.getClass()) + && valueToAdd instanceof String) { + target.add(new RecordHeader(key, ((String) valueToAdd).getBytes(getCharset()))); + className = JAVA_LANG_STRING; + } + else { + target.add(new RecordHeader(key, headerObjectMapper.writeValueAsBytes(valueToAdd))); + } + jsonHeaders.put(key, className); + } + catch (Exception e) { + logger.debug(e, () -> "Could not map " + key + " with type " + rawValue.getClass().getName()); + } + } + } + }); + if (jsonHeaders.size() > 0) { + try { + target.add(new RecordHeader(JSON_TYPES, headerObjectMapper.writeValueAsBytes(jsonHeaders))); + } + catch (IllegalStateException | JsonProcessingException e) { + logger.error(e, "Could not add json types header"); + } + } + } + + @Override + public void toHeaders(Headers source, final Map headers) { + final Map jsonTypes = decodeJsonTypes(source); + source.forEach(header -> { + if (!(header.key().equals(JSON_TYPES))) { + if (jsonTypes != null && jsonTypes.containsKey(header.key())) { + String requestedType = jsonTypes.get(header.key()); + populateJsonValueHeader(header, requestedType, headers); + } + else { + headers.put(header.key(), headerValueToAddIn(header)); + } + } + }); + } + + private void populateJsonValueHeader(Header header, String requestedType, Map headers) { + Class type = Object.class; + boolean trusted = false; + try { + trusted = trusted(requestedType); + if (trusted) { + type = ClassUtils.forName(requestedType, null); + } + } + catch (Exception e) { + logger.error(e, () -> "Could not load class for header: " + header.key()); + } + if (String.class.equals(type) && (header.value().length == 0 || header.value()[0] != '"')) { + headers.put(header.key(), new String(header.value(), getCharset())); + } + else { + if (trusted) { + try { + Object value = decodeValue(header, type); + headers.put(header.key(), value); + } + catch (IOException e) { + logger.error(e, () -> + "Could not decode json type: " + new String(header.value()) + " for key: " + + header.key()); + headers.put(header.key(), header.value()); + } + } + else { + headers.put(header.key(), new NonTrustedHeaderType(header.value(), requestedType)); + } + } + } + + private Object decodeValue(Header h, Class type) throws IOException, LinkageError { + ObjectMapper headerObjectMapper = getObjectMapper(); + Object value = headerObjectMapper.readValue(h.value(), type); + if (type.equals(NonTrustedHeaderType.class)) { + // Upstream NTHT propagated; may be trusted here... + NonTrustedHeaderType nth = (NonTrustedHeaderType) value; + if (trusted(nth.getUntrustedType())) { + try { + value = headerObjectMapper.readValue(nth.getHeaderValue(), + ClassUtils.forName(nth.getUntrustedType(), null)); + } + catch (Exception e) { + logger.error(e, () -> "Could not decode header: " + nth); + } + } + } + return value; + } + + @SuppressWarnings("unchecked") + @Nullable + private Map decodeJsonTypes(Headers source) { + Map types = null; + Header jsonTypes = source.lastHeader(JSON_TYPES); + if (jsonTypes != null) { + ObjectMapper headerObjectMapper = getObjectMapper(); + try { + types = headerObjectMapper.readValue(jsonTypes.value(), Map.class); + } + catch (IOException e) { + logger.error(e, () -> "Could not decode json types: " + new String(jsonTypes.value())); + } + } + return types; + } + + protected boolean trusted(String requestedType) { + if (requestedType.equals(NonTrustedHeaderType.class.getName())) { + return true; + } + if (!this.trustedPackages.isEmpty()) { + int lastDot = requestedType.lastIndexOf('.'); + if (lastDot < 0) { + return false; + } + String packageName = requestedType.substring(0, lastDot); + for (String trustedPackage : this.trustedPackages) { + if (packageName.equals(trustedPackage) || packageName.startsWith(trustedPackage + ".")) { + return true; + } + } + return false; + } + return true; + } + + /** + * Add patterns for headers that should never be mapped. + * @param patterns the patterns. + * @return the modified patterns. + * @since 3.0.2 + */ + public static String[] addNeverHeaderPatterns(List patterns) { + List patternsToUse = new LinkedList<>(patterns); + patternsToUse.add(0, NEVER_NATIVE_HEADERS_PRESENT); + patternsToUse.add(0, NEVER_DELIVERY_ATTEMPT); + patternsToUse.add(0, NEVER_TIMESTAMP); + patternsToUse.add(0, NEVER_ID); + return patternsToUse.toArray(new String[0]); + } + + /** + * Remove never headers. + * @param headers the headers from which to remove the never headers. + * @since 3.0.2 + */ + public static void removeNeverHeaders(Headers headers) { + headers.remove(MessageHeaders.ID); + headers.remove(MessageHeaders.TIMESTAMP); + headers.remove(IntegrationMessageHeaderAccessor.DELIVERY_ATTEMPT); + headers.remove(BinderHeaders.NATIVE_HEADERS_PRESENT); + } + + /** + * The {@link StdNodeBasedDeserializer} extension for {@link MimeType} deserialization. + * It is presented here for backward compatibility when older producers send {@link MimeType} + * headers as serialization version. + */ + private class MimeTypeJsonDeserializer extends StdNodeBasedDeserializer { + + private static final long serialVersionUID = 1L; + + MimeTypeJsonDeserializer() { + super(MimeType.class); + } + + @Override + public MimeType convert(JsonNode root, DeserializationContext ctxt) throws IOException { + if (root instanceof TextNode) { + return MimeType.valueOf(root.asText()); + } + else { + JsonNode type = root.get("type"); + JsonNode subType = root.get("subtype"); + JsonNode parameters = root.get("parameters"); + Map params = + BinderHeaderMapper.this.objectMapper.readValue(parameters.traverse(), + TypeFactory.defaultInstance() + .constructMapType(HashMap.class, String.class, String.class)); + return new MimeType(type.asText(), subType.asText(), params); + } + } + + } + + /** + * Represents a header that could not be decoded due to an untrusted type. + */ + public static class NonTrustedHeaderType { + + private byte[] headerValue; + + private String untrustedType; + + public NonTrustedHeaderType() { + super(); + } + + NonTrustedHeaderType(byte[] headerValue, String untrustedType) { // NOSONAR + this.headerValue = headerValue; // NOSONAR + this.untrustedType = untrustedType; + } + + + public void setHeaderValue(byte[] headerValue) { // NOSONAR + this.headerValue = headerValue; // NOSONAR array reference + } + + public byte[] getHeaderValue() { + return this.headerValue; // NOSONAR + } + + public void setUntrustedType(String untrustedType) { + this.untrustedType = untrustedType; + } + + public String getUntrustedType() { + return this.untrustedType; + } + + @Override + public String toString() { + try { + return "NonTrustedHeaderType [headerValue=" + new String(this.headerValue, StandardCharsets.UTF_8) + + ", untrustedType=" + this.untrustedType + "]"; + } + catch (@SuppressWarnings("unused") Exception e) { + return "NonTrustedHeaderType [headerValue=" + Arrays.toString(this.headerValue) + ", untrustedType=" + + this.untrustedType + "]"; + } + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderEnvironmentPostProcessor.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderEnvironmentPostProcessor.java new file mode 100644 index 000000000..7fe2ef06b --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderEnvironmentPostProcessor.java @@ -0,0 +1,82 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.env.EnvironmentPostProcessor; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.MapPropertySource; + +/** + * An {@link EnvironmentPostProcessor} that sets some common configuration properties (log + * config etc.,) for Kafka binder. + * + * @author Ilayaperumal Gopinathan + */ +public class KafkaBinderEnvironmentPostProcessor implements EnvironmentPostProcessor { + + private static final String SPRING_KAFKA = "spring.kafka"; + + private static final String SPRING_KAFKA_PRODUCER = SPRING_KAFKA + ".producer"; + + private static final String SPRING_KAFKA_CONSUMER = SPRING_KAFKA + ".consumer"; + + private static final String SPRING_KAFKA_PRODUCER_KEY_SERIALIZER = SPRING_KAFKA_PRODUCER + + "." + "keySerializer"; + + private static final String SPRING_KAFKA_PRODUCER_VALUE_SERIALIZER = SPRING_KAFKA_PRODUCER + + "." + "valueSerializer"; + + private static final String SPRING_KAFKA_CONSUMER_KEY_DESERIALIZER = SPRING_KAFKA_CONSUMER + + "." + "keyDeserializer"; + + private static final String SPRING_KAFKA_CONSUMER_VALUE_DESERIALIZER = SPRING_KAFKA_CONSUMER + + "." + "valueDeserializer"; + + private static final String KAFKA_BINDER_DEFAULT_PROPERTIES = "kafkaBinderDefaultProperties"; + + @Override + public void postProcessEnvironment(ConfigurableEnvironment environment, + SpringApplication application) { + if (!environment.getPropertySources().contains(KAFKA_BINDER_DEFAULT_PROPERTIES)) { + Map kafkaBinderDefaultProperties = new HashMap<>(); + kafkaBinderDefaultProperties.put("logging.level.org.I0Itec.zkclient", + "ERROR"); + kafkaBinderDefaultProperties.put("logging.level.kafka.server.KafkaConfig", + "ERROR"); + kafkaBinderDefaultProperties + .put("logging.level.kafka.admin.AdminClient.AdminConfig", "ERROR"); + kafkaBinderDefaultProperties.put(SPRING_KAFKA_PRODUCER_KEY_SERIALIZER, + ByteArraySerializer.class.getName()); + kafkaBinderDefaultProperties.put(SPRING_KAFKA_PRODUCER_VALUE_SERIALIZER, + ByteArraySerializer.class.getName()); + kafkaBinderDefaultProperties.put(SPRING_KAFKA_CONSUMER_KEY_DESERIALIZER, + ByteArrayDeserializer.class.getName()); + kafkaBinderDefaultProperties.put(SPRING_KAFKA_CONSUMER_VALUE_DESERIALIZER, + ByteArrayDeserializer.class.getName()); + environment.getPropertySources().addLast(new MapPropertySource( + KAFKA_BINDER_DEFAULT_PROPERTIES, kafkaBinderDefaultProperties)); + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealth.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealth.java new file mode 100644 index 000000000..dd6e6b6b1 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealth.java @@ -0,0 +1,29 @@ +/* + * Copyright 2022-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import org.springframework.boot.actuate.health.HealthIndicator; + +/** + * Marker interface used for custom KafkaBinderHealth indicator implementations. + * + * @author Soby Chacko + * @since 3.2.2 + */ +public interface KafkaBinderHealth extends HealthIndicator { + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicator.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicator.java new file mode 100644 index 000000000..e35d15290 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicator.java @@ -0,0 +1,225 @@ +/* + * Copyright 2016-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.common.PartitionInfo; + +import org.springframework.beans.factory.DisposableBean; +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.Status; +import org.springframework.boot.actuate.health.StatusAggregator; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.listener.AbstractMessageListenerContainer; +import org.springframework.scheduling.concurrent.CustomizableThreadFactory; + +/** + * Health indicator for Kafka. + * + * @author Ilayaperumal Gopinathan + * @author Marius Bogoevici + * @author Henryk Konsek + * @author Gary Russell + * @author Laur Aliste + * @author Soby Chacko + * @author Vladislav Fefelov + * @author Chukwubuikem Ume-Ugwa + * @author Taras Danylchuk + */ +public class KafkaBinderHealthIndicator implements KafkaBinderHealth, DisposableBean { + + private static final int DEFAULT_TIMEOUT = 60; + + private final ExecutorService executor = Executors.newSingleThreadExecutor( + new CustomizableThreadFactory("kafka-binder-health-")); + + private final KafkaMessageChannelBinder binder; + + private final ConsumerFactory consumerFactory; + + private int timeout = DEFAULT_TIMEOUT; + + private Consumer metadataConsumer; + + private boolean considerDownWhenAnyPartitionHasNoLeader; + + public KafkaBinderHealthIndicator(KafkaMessageChannelBinder binder, + ConsumerFactory consumerFactory) { + this.binder = binder; + this.consumerFactory = consumerFactory; + } + + /** + * Set the timeout in seconds to retrieve health information. + * @param timeout the timeout - default 60. + */ + public void setTimeout(int timeout) { + this.timeout = timeout; + } + + public void setConsiderDownWhenAnyPartitionHasNoLeader(boolean considerDownWhenAnyPartitionHasNoLeader) { + this.considerDownWhenAnyPartitionHasNoLeader = considerDownWhenAnyPartitionHasNoLeader; + } + + @Override + public Health health() { + Health topicsHealth = safelyBuildTopicsHealth(); + Health listenerContainersHealth = buildListenerContainersHealth(); + return merge(topicsHealth, listenerContainersHealth); + } + + private Health merge(Health topicsHealth, Health listenerContainersHealth) { + Status aggregatedStatus = StatusAggregator.getDefault() + .getAggregateStatus(topicsHealth.getStatus(), listenerContainersHealth.getStatus()); + Map aggregatedDetails = new HashMap<>(); + aggregatedDetails.putAll(topicsHealth.getDetails()); + aggregatedDetails.putAll(listenerContainersHealth.getDetails()); + return Health.status(aggregatedStatus).withDetails(aggregatedDetails).build(); + } + + private Health safelyBuildTopicsHealth() { + Future future = executor.submit(this::buildTopicsHealth); + try { + return future.get(this.timeout, TimeUnit.SECONDS); + } + catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return Health.down() + .withDetail("Interrupted while waiting for partition information in", + this.timeout + " seconds") + .build(); + } + catch (ExecutionException ex) { + return Health.down(ex).build(); + } + catch (TimeoutException ex) { + return Health.down().withDetail("Failed to retrieve partition information in", + this.timeout + " seconds").build(); + } + } + + private void initMetadataConsumer() { + if (this.metadataConsumer == null) { + this.metadataConsumer = this.consumerFactory.createConsumer(); + } + } + + private Health buildTopicsHealth() { + try { + initMetadataConsumer(); + Set downMessages = new HashSet<>(); + Set checkedTopics = new HashSet<>(); + final Map topicsInUse = KafkaBinderHealthIndicator.this.binder + .getTopicsInUse(); + if (topicsInUse.isEmpty()) { + try { + this.metadataConsumer.listTopics(Duration.ofSeconds(this.timeout)); + } + catch (Exception e) { + return Health.down().withDetail("No topic information available", + "Kafka broker is not reachable").build(); + } + return Health.unknown().withDetail("No bindings found", + "Kafka binder may not be bound to destinations on the broker").build(); + } + else { + for (String topic : topicsInUse.keySet()) { + KafkaMessageChannelBinder.TopicInformation topicInformation = topicsInUse + .get(topic); + if (!topicInformation.isTopicPattern()) { + List partitionInfos = this.metadataConsumer + .partitionsFor(topic); + for (PartitionInfo partitionInfo : partitionInfos) { + if (topicInformation.getPartitionInfos() + .contains(partitionInfo) + && partitionInfo.leader() == null || + (partitionInfo.leader() != null && partitionInfo.leader().id() == -1)) { + downMessages.add(partitionInfo.toString()); + } + else if (this.considerDownWhenAnyPartitionHasNoLeader && + partitionInfo.leader() == null || (partitionInfo.leader() != null && partitionInfo.leader().id() == -1)) { + downMessages.add(partitionInfo.toString()); + } + } + checkedTopics.add(topic); + } + } + } + if (downMessages.isEmpty()) { + return Health.up().withDetail("topicsInUse", checkedTopics).build(); + } + else { + return Health.down() + .withDetail("Following partitions in use have no leaders: ", + downMessages.toString()) + .build(); + } + } + catch (Exception ex) { + return Health.down(ex).build(); + } + } + + private Health buildListenerContainersHealth() { + List> listenerContainers = binder.getKafkaMessageListenerContainers(); + if (listenerContainers.isEmpty()) { + return Health.unknown().build(); + } + + Status status = Status.UP; + List> containersDetails = new ArrayList<>(); + + for (AbstractMessageListenerContainer container : listenerContainers) { + Map containerDetails = new HashMap<>(); + boolean isRunning = container.isRunning(); + boolean isOk = container.isInExpectedState(); + if (!isOk) { + status = Status.DOWN; + } + containerDetails.put("isRunning", isRunning); + containerDetails.put("isStoppedAbnormally", !isRunning && !isOk); + containerDetails.put("isPaused", container.isContainerPaused()); + containerDetails.put("listenerId", container.getListenerId()); + containerDetails.put("groupId", container.getGroupId()); + + containersDetails.add(containerDetails); + } + return Health.status(status) + .withDetail("listenerContainers", containersDetails) + .build(); + } + + @Override + public void destroy() { + executor.shutdown(); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderMetrics.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderMetrics.java new file mode 100644 index 000000000..f07e9aee1 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderMetrics.java @@ -0,0 +1,255 @@ +/* + * Copyright 2016-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.binder.MeterBinder; +import io.micrometer.core.instrument.noop.NoopGauge; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; + +import org.springframework.cloud.stream.binder.BindingCreatedEvent; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.context.ApplicationListener; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.lang.Nullable; +import org.springframework.util.ObjectUtils; + +/** + * Metrics for Kafka binder. + * + * @author Henryk Konsek + * @author Soby Chacko + * @author Artem Bilan + * @author Oleg Zhurakousky + * @author Jon Schneider + * @author Thomas Cheyney + * @author Gary Russell + */ +public class KafkaBinderMetrics + implements MeterBinder, ApplicationListener { + + private static final int DEFAULT_TIMEOUT = 5; + + private static final int DELAY_BETWEEN_TASK_EXECUTION = 60; + + private static final Log LOG = LogFactory.getLog(KafkaBinderMetrics.class); + + /** + * Offset lag micrometer metric name. This can be used for meter filtering. + */ + public static final String OFFSET_LAG_METRIC_NAME = "spring.cloud.stream.binder.kafka.offset"; + + private final KafkaMessageChannelBinder binder; + + private final KafkaBinderConfigurationProperties binderConfigurationProperties; + + private ConsumerFactory defaultConsumerFactory; + + private final MeterRegistry meterRegistry; + + private Map> metadataConsumers; + + private int timeout = DEFAULT_TIMEOUT; + + ScheduledExecutorService scheduler; + + Map unconsumedMessages = new ConcurrentHashMap<>(); + + public KafkaBinderMetrics(KafkaMessageChannelBinder binder, + KafkaBinderConfigurationProperties binderConfigurationProperties, + ConsumerFactory defaultConsumerFactory, + @Nullable MeterRegistry meterRegistry) { + + this.binder = binder; + this.binderConfigurationProperties = binderConfigurationProperties; + this.defaultConsumerFactory = defaultConsumerFactory; + this.meterRegistry = meterRegistry; + this.metadataConsumers = new ConcurrentHashMap<>(); + } + + public KafkaBinderMetrics(KafkaMessageChannelBinder binder, + KafkaBinderConfigurationProperties binderConfigurationProperties) { + + this(binder, binderConfigurationProperties, null, null); + } + + public void setTimeout(int timeout) { + this.timeout = timeout; + } + + @Override + public void bindTo(MeterRegistry registry) { + /** + * We can't just replace one scheduler with another. + * Before and even after the old one is gathered by GC, it's threads still exist, consume memory and CPU resources to switch contexts. + * Theoretically, as a result of processing n topics, there will be about (1+n)*n/2 threads simultaneously at the same time. + */ + if (this.scheduler != null) { + LOG.info("Try to shutdown the old scheduler with " + ((ScheduledThreadPoolExecutor) scheduler).getPoolSize() + " threads"); + this.scheduler.shutdown(); + } + + this.scheduler = Executors.newScheduledThreadPool(this.binder.getTopicsInUse().size()); + + for (Map.Entry topicInfo : this.binder + .getTopicsInUse().entrySet()) { + + if (!topicInfo.getValue().isConsumerTopic()) { + continue; + } + + String topic = topicInfo.getKey(); + String group = topicInfo.getValue().getConsumerGroup(); + + final Gauge register = Gauge.builder(OFFSET_LAG_METRIC_NAME, this, + (o) -> computeAndGetUnconsumedMessages(topic, group)).tag("group", group) + .tag("topic", topic) + .description("Unconsumed messages for a particular group and topic") + .register(registry); + + if (!(register instanceof NoopGauge)) { + //Schedule a task to compute the unconsumed messages for this group/topic every minute. + this.scheduler.scheduleWithFixedDelay(computeUnconsumedMessagesRunnable(topic, group, this.metadataConsumers), + 10, DELAY_BETWEEN_TASK_EXECUTION, TimeUnit.SECONDS); + } + } + } + + private Runnable computeUnconsumedMessagesRunnable(String topic, String group, Map> metadataConsumers) { + return () -> { + try { + long lag = findTotalTopicGroupLag(topic, group, this.metadataConsumers); + this.unconsumedMessages.put(topic + "-" + group, lag); + } + catch (Exception ex) { + LOG.debug("Cannot generate metric for topic: " + topic, ex); + } + }; + } + + private long computeAndGetUnconsumedMessages(String topic, String group) { + ExecutorService exec = Executors.newCachedThreadPool(); + Future future = exec.submit(() -> { + + long lag = 0; + try { + lag = findTotalTopicGroupLag(topic, group, this.metadataConsumers); + } + catch (Exception ex) { + LOG.debug("Cannot generate metric for topic: " + topic, ex); + } + return lag; + }); + try { + return future.get(this.timeout, TimeUnit.SECONDS); + } + catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return this.unconsumedMessages.getOrDefault(topic + "-" + group, 0L); + } + catch (ExecutionException | TimeoutException ex) { + return this.unconsumedMessages.getOrDefault(topic + "-" + group, 0L); + } + finally { + exec.shutdownNow(); + } + } + + private long findTotalTopicGroupLag(String topic, String group, Map> metadataConsumers) { + long lag = 0; + Consumer metadataConsumer = metadataConsumers.computeIfAbsent( + group, + (g) -> createConsumerFactory().createConsumer(g, "monitoring")); + List partitionInfos = metadataConsumer + .partitionsFor(topic); + List topicPartitions = new LinkedList<>(); + for (PartitionInfo partitionInfo : partitionInfos) { + topicPartitions.add(new TopicPartition(partitionInfo.topic(), + partitionInfo.partition())); + } + + Map endOffsets = metadataConsumer + .endOffsets(topicPartitions); + + final Map committedOffsets = metadataConsumer.committed(endOffsets.keySet()); + + for (Map.Entry endOffset : endOffsets + .entrySet()) { + OffsetAndMetadata current = committedOffsets.get(endOffset.getKey()); + lag += endOffset.getValue(); + if (current != null) { + lag -= current.offset(); + } + } + return lag; + } + + private synchronized ConsumerFactory createConsumerFactory() { + if (this.defaultConsumerFactory == null) { + Map props = new HashMap<>(); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + ByteArrayDeserializer.class); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + ByteArrayDeserializer.class); + Map mergedConfig = this.binderConfigurationProperties + .mergedConsumerConfiguration(); + if (!ObjectUtils.isEmpty(mergedConfig)) { + props.putAll(mergedConfig); + } + if (!props.containsKey(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)) { + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + this.binderConfigurationProperties + .getKafkaConnectionString()); + } + this.defaultConsumerFactory = new DefaultKafkaConsumerFactory<>( + props); + } + return this.defaultConsumerFactory; + } + + @Override + public void onApplicationEvent(BindingCreatedEvent event) { + if (this.meterRegistry != null) { + // It is safe to call bindTo multiple times, since meters are idempotent when called with the same arguments + this.bindTo(this.meterRegistry); + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBindingRebalanceListener.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBindingRebalanceListener.java new file mode 100644 index 000000000..4bcc62a49 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBindingRebalanceListener.java @@ -0,0 +1,74 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.Collection; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.common.TopicPartition; + +/** + * A rebalance listener that provides access to the binding name consumer object. It can + * be used to perform seek operations on the consumer after a rebalance. + * + * @author Gary Russell + * @since 2.1 + * + */ +public interface KafkaBindingRebalanceListener { + + /** + * Invoked by the container before any pending offsets are committed. + * @param bindingName the name of the binding. + * @param consumer the consumer. + * @param partitions the partitions. + */ + default void onPartitionsRevokedBeforeCommit(String bindingName, + Consumer consumer, Collection partitions) { + // do nothing + } + + /** + * Invoked by the container after any pending offsets are committed. + * @param bindingName the name of the binding. + * @param consumer the consumer. + * @param partitions the partitions. + */ + default void onPartitionsRevokedAfterCommit(String bindingName, + Consumer consumer, Collection partitions) { + // do nothing + } + + /** + * Invoked when partitions are initially assigned or after a rebalance. Applications + * might only want to perform seek operations on an initial assignment. While the + * 'initial' argument is true for each thread (when concurrency is greater than 1), + * implementations should keep track of exactly which partitions have been sought. + * There is a race in that a rebalance could occur during startup and so a topic/ + * partition that has been sought on one thread may be re-assigned to another + * thread and you may not wish to re-seek it at that time. + * @param bindingName the name of the binding. + * @param consumer the consumer. + * @param partitions the partitions. + * @param initial true if this is the initial assignment on the current thread. + */ + default void onPartitionsAssigned(String bindingName, Consumer consumer, + Collection partitions, boolean initial) { + // do nothing + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaExpressionEvaluatingInterceptor.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaExpressionEvaluatingInterceptor.java new file mode 100644 index 000000000..070329092 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaExpressionEvaluatingInterceptor.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.Expression; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.support.ChannelInterceptor; +import org.springframework.util.Assert; + +/** + * Interceptor to evaluate expressions for outbound messages before serialization. + * + * @author Gary Russell + * @since 3.0 + * + */ +public class KafkaExpressionEvaluatingInterceptor implements ChannelInterceptor { + + /** + * Name for the evaluated message key header. + */ + public static final String MESSAGE_KEY_HEADER = "scst_messageKey"; + + private final Expression messageKeyExpression; + + private final EvaluationContext evaluationContext; + + /** + * Construct an instance with the provided expressions and evaluation context. At + * least one expression muse be non-null. + * @param messageKeyExpression the routing key expression. + * @param evaluationContext the evaluation context. + */ + public KafkaExpressionEvaluatingInterceptor(Expression messageKeyExpression, EvaluationContext evaluationContext) { + Assert.notNull(messageKeyExpression != null, "A message key expression is required"); + Assert.notNull(evaluationContext, "the 'evaluationContext' cannot be null"); + this.messageKeyExpression = messageKeyExpression; + this.evaluationContext = evaluationContext; + } + + @Override + public Message preSend(Message message, MessageChannel channel) { + MessageBuilder builder = MessageBuilder.fromMessage(message); + if (this.messageKeyExpression != null) { + builder.setHeader(MESSAGE_KEY_HEADER, + this.messageKeyExpression.getValue(this.evaluationContext, message)); + } + return builder.build(); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java new file mode 100644 index 000000000..6a4973def --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java @@ -0,0 +1,1715 @@ +/* + * Copyright 2014-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import java.util.function.Predicate; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.NoSuchBeanDefinitionException; +import org.springframework.cloud.stream.binder.AbstractMessageChannelBinder; +import org.springframework.cloud.stream.binder.BinderHeaders; +import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider; +import org.springframework.cloud.stream.binder.DefaultPollableMessageSource; +import org.springframework.cloud.stream.binder.EmbeddedHeaderUtils; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder; +import org.springframework.cloud.stream.binder.HeaderMode; +import org.springframework.cloud.stream.binder.MessageValues; +import org.springframework.cloud.stream.binder.kafka.config.ClientFactoryCustomizer; +import org.springframework.cloud.stream.binder.kafka.config.ConsumerConfigCustomizer; +import org.springframework.cloud.stream.binder.kafka.config.ProducerConfigCustomizer; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties.StandardHeaders; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaExtendedBindingProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.cloud.stream.binder.kafka.utils.DlqDestinationResolver; +import org.springframework.cloud.stream.binder.kafka.utils.DlqPartitionFunction; +import org.springframework.cloud.stream.binding.DefaultPartitioningInterceptor; +import org.springframework.cloud.stream.binding.MessageConverterConfigurer.PartitioningInterceptor; +import org.springframework.cloud.stream.config.ListenerContainerCustomizer; +import org.springframework.cloud.stream.config.MessageSourceCustomizer; +import org.springframework.cloud.stream.provisioning.ConsumerDestination; +import org.springframework.cloud.stream.provisioning.ProducerDestination; +import org.springframework.context.support.AbstractApplicationContext; +import org.springframework.expression.Expression; +import org.springframework.expression.common.LiteralExpression; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.integration.IntegrationMessageHeaderAccessor; +import org.springframework.integration.StaticMessageHeaderAccessor; +import org.springframework.integration.acks.AcknowledgmentCallback; +import org.springframework.integration.channel.AbstractMessageChannel; +import org.springframework.integration.core.MessageProducer; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter.ListenerMode; +import org.springframework.integration.kafka.inbound.KafkaMessageSource; +import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler; +import org.springframework.integration.kafka.support.RawRecordHeaderErrorMessageStrategy; +import org.springframework.integration.support.ErrorMessageStrategy; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.AbstractMessageListenerContainer; +import org.springframework.kafka.listener.CommonErrorHandler; +import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; +import org.springframework.kafka.listener.ConsumerAwareRebalanceListener; +import org.springframework.kafka.listener.ConsumerProperties; +import org.springframework.kafka.listener.ContainerProperties; +import org.springframework.kafka.listener.DefaultAfterRollbackProcessor; +import org.springframework.kafka.listener.DefaultErrorHandler; +import org.springframework.kafka.support.Acknowledgment; +import org.springframework.kafka.support.ExponentialBackOffWithMaxRetries; +import org.springframework.kafka.support.KafkaHeaderMapper; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.kafka.support.ProducerListener; +import org.springframework.kafka.support.SendResult; +import org.springframework.kafka.support.TopicPartitionOffset; +import org.springframework.kafka.support.TopicPartitionOffset.SeekPosition; +import org.springframework.kafka.support.converter.MessageConverter; +import org.springframework.kafka.support.converter.MessagingMessageConverter; +import org.springframework.kafka.support.converter.RecordMessageConverter; +import org.springframework.kafka.transaction.KafkaAwareTransactionManager; +import org.springframework.kafka.transaction.KafkaTransactionManager; +import org.springframework.lang.Nullable; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.MessagingException; +import org.springframework.messaging.support.ChannelInterceptor; +import org.springframework.messaging.support.ErrorMessage; +import org.springframework.messaging.support.InterceptableChannel; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ObjectUtils; +import org.springframework.util.StringUtils; +import org.springframework.util.backoff.BackOff; +import org.springframework.util.backoff.ExponentialBackOff; +import org.springframework.util.backoff.FixedBackOff; +import org.springframework.util.concurrent.ListenableFuture; +import org.springframework.util.concurrent.ListenableFutureCallback; + +/** + * A {@link org.springframework.cloud.stream.binder.Binder} that uses Kafka as the + * underlying middleware. + * + * @author Eric Bottard + * @author Marius Bogoevici + * @author Ilayaperumal Gopinathan + * @author David Turanski + * @author Gary Russell + * @author Mark Fisher + * @author Soby Chacko + * @author Henryk Konsek + * @author Doug Saus + * @author Lukasz Kaminski + * @author Taras Danylchuk + */ +public class KafkaMessageChannelBinder extends + // @checkstyle:off + AbstractMessageChannelBinder, ExtendedProducerProperties, KafkaTopicProvisioner> + // @checkstyle:on + implements + ExtendedPropertiesBinder { + + /** + * Kafka header for x-exception-fqcn. + */ + public static final String X_EXCEPTION_FQCN = "x-exception-fqcn"; + + /** + * Kafka header for x-exception-stacktrace. + */ + public static final String X_EXCEPTION_STACKTRACE = "x-exception-stacktrace"; + + /** + * Kafka header for x-exception-message. + */ + public static final String X_EXCEPTION_MESSAGE = "x-exception-message"; + + /** + * Kafka header for x-original-topic. + */ + public static final String X_ORIGINAL_TOPIC = "x-original-topic"; + + /** + * Kafka header for x-original-partition. + */ + public static final String X_ORIGINAL_PARTITION = "x-original-partition"; + + /** + * Kafka header for x-original-offset. + */ + public static final String X_ORIGINAL_OFFSET = "x-original-offset"; + + /** + * Kafka header for x-original-timestamp. + */ + public static final String X_ORIGINAL_TIMESTAMP = "x-original-timestamp"; + + /** + * Kafka header for x-original-timestamp-type. + */ + public static final String X_ORIGINAL_TIMESTAMP_TYPE = "x-original-timestamp-type"; + + private static final ThreadLocal bindingNameHolder = new ThreadLocal<>(); + + private static final Pattern interceptorNeededPattern = Pattern.compile("(payload|#root|#this)"); + + private static final SpelExpressionParser PARSER = new SpelExpressionParser(); + + private final KafkaBinderConfigurationProperties configurationProperties; + + private final Map topicsInUse = new ConcurrentHashMap<>(); + + private final KafkaTransactionManager transactionManager; + + private final TransactionTemplate transactionTemplate; + + private KafkaBindingRebalanceListener rebalanceListener; + + private DlqPartitionFunction dlqPartitionFunction; + + private DlqDestinationResolver dlqDestinationResolver; + + private final Map ackModeInfo = new ConcurrentHashMap<>(); + + private ProducerListener producerListener; + + private KafkaExtendedBindingProperties extendedBindingProperties = new KafkaExtendedBindingProperties(); + + private ClientFactoryCustomizer clientFactoryCustomizer; + + private ProducerConfigCustomizer producerConfigCustomizer; + + private ConsumerConfigCustomizer consumerConfigCustomizer; + + private final List> kafkaMessageListenerContainers = new ArrayList<>(); + + public KafkaMessageChannelBinder( + KafkaBinderConfigurationProperties configurationProperties, + KafkaTopicProvisioner provisioningProvider) { + + this(configurationProperties, provisioningProvider, null, null, null, + null, null); + } + + public KafkaMessageChannelBinder( + KafkaBinderConfigurationProperties configurationProperties, + KafkaTopicProvisioner provisioningProvider, + ListenerContainerCustomizer> containerCustomizer, + KafkaBindingRebalanceListener rebalanceListener) { + + this(configurationProperties, provisioningProvider, containerCustomizer, null, rebalanceListener, + null, null); + } + + public KafkaMessageChannelBinder( + KafkaBinderConfigurationProperties configurationProperties, + KafkaTopicProvisioner provisioningProvider, + ListenerContainerCustomizer> containerCustomizer, + MessageSourceCustomizer> sourceCustomizer, + KafkaBindingRebalanceListener rebalanceListener, + DlqPartitionFunction dlqPartitionFunction, + DlqDestinationResolver dlqDestinationResolver) { + + super(headersToMap(configurationProperties), provisioningProvider, + containerCustomizer, sourceCustomizer); + this.configurationProperties = configurationProperties; + String txId = configurationProperties.getTransaction().getTransactionIdPrefix(); + if (StringUtils.hasText(txId)) { + this.transactionManager = new KafkaTransactionManager<>(getProducerFactory( + txId, new ExtendedProducerProperties<>(configurationProperties + .getTransaction().getProducer().getExtension()), txId + ".producer", null)); + this.transactionTemplate = new TransactionTemplate(this.transactionManager); + } + else { + this.transactionManager = null; + this.transactionTemplate = null; + } + this.rebalanceListener = rebalanceListener; + this.dlqPartitionFunction = dlqPartitionFunction; + this.dlqDestinationResolver = dlqDestinationResolver; + } + + private static String[] headersToMap( + KafkaBinderConfigurationProperties configurationProperties) { + String[] headersToMap; + if (ObjectUtils.isEmpty(configurationProperties.getHeaders())) { + headersToMap = BinderHeaders.STANDARD_HEADERS; + } + else { + String[] combinedHeadersToMap = Arrays.copyOfRange( + BinderHeaders.STANDARD_HEADERS, 0, + BinderHeaders.STANDARD_HEADERS.length + + configurationProperties.getHeaders().length); + System.arraycopy(configurationProperties.getHeaders(), 0, + combinedHeadersToMap, BinderHeaders.STANDARD_HEADERS.length, + configurationProperties.getHeaders().length); + headersToMap = combinedHeadersToMap; + } + return headersToMap; + } + + public void setExtendedBindingProperties( + KafkaExtendedBindingProperties extendedBindingProperties) { + this.extendedBindingProperties = extendedBindingProperties; + } + + public void setProducerListener(ProducerListener producerListener) { + this.producerListener = producerListener; + } + + public void setClientFactoryCustomizer(ClientFactoryCustomizer customizer) { + this.clientFactoryCustomizer = customizer; + } + + public void setRebalanceListener(KafkaBindingRebalanceListener rebalanceListener) { + this.rebalanceListener = rebalanceListener; + } + + public void setDlqPartitionFunction(DlqPartitionFunction dlqPartitionFunction) { + this.dlqPartitionFunction = dlqPartitionFunction; + } + + public void setDlqDestinationResolver(DlqDestinationResolver dlqDestinationResolver) { + this.dlqDestinationResolver = dlqDestinationResolver; + } + + Map getTopicsInUse() { + return this.topicsInUse; + } + + @Override + public KafkaConsumerProperties getExtendedConsumerProperties(String channelName) { + bindingNameHolder.set(channelName); + return this.extendedBindingProperties.getExtendedConsumerProperties(channelName); + } + + @Override + public KafkaProducerProperties getExtendedProducerProperties(String channelName) { + bindingNameHolder.set(channelName); + return this.extendedBindingProperties.getExtendedProducerProperties(channelName); + } + + @Override + public String getDefaultsPrefix() { + return this.extendedBindingProperties.getDefaultsPrefix(); + } + + @Override + public Class getExtendedPropertiesEntryClass() { + return this.extendedBindingProperties.getExtendedPropertiesEntryClass(); + } + + /** + * Return a reference to the binder's transaction manager's producer factory (if + * configured). Use this to create a transaction manager in a bean definition when you + * wish to use producer-only transactions. + * @return the transaction manager, or null. + */ + @Nullable + public ProducerFactory getTransactionalProducerFactory() { + return this.transactionManager == null ? null : this.transactionManager.getProducerFactory(); + } + + @Override + protected MessageHandler createProducerMessageHandler( + final ProducerDestination destination, + ExtendedProducerProperties producerProperties, + MessageChannel errorChannel) throws Exception { + throw new IllegalStateException( + "The abstract binder should not call this method"); + } + + @Override + protected MessageHandler createProducerMessageHandler( + final ProducerDestination destination, + ExtendedProducerProperties producerProperties, + MessageChannel channel, MessageChannel errorChannel) throws Exception { + /* + * IMPORTANT: With a transactional binder, individual producer properties for + * Kafka are ignored; the global binder + * (spring.cloud.stream.kafka.binder.transaction.producer.*) properties are used + * instead, for all producers. A binder is transactional when + * 'spring.cloud.stream.kafka.binder.transaction.transaction-id-prefix' has text. + * Individual bindings can override the binder's transaction manager. + */ + KafkaAwareTransactionManager transMan = transactionManager( + producerProperties.getExtension().getTransactionManager()); + final ProducerFactory producerFB = transMan != null + ? transMan.getProducerFactory() + : getProducerFactory(null, producerProperties, destination.getName() + ".producer", + destination.getName()); + Collection partitions = provisioningProvider.getPartitionsForTopic( + producerProperties.getPartitionCount(), false, () -> { + Producer producer = producerFB.createProducer(); + List partitionsFor = producer + .partitionsFor(destination.getName()); + producer.close(); + return partitionsFor; + }, destination.getName()); + this.topicsInUse.put(destination.getName(), + new TopicInformation(null, partitions, false)); + if (producerProperties.isPartitioned() + && producerProperties.getPartitionCount() < partitions.size()) { + if (this.logger.isInfoEnabled()) { + this.logger.info("The `partitionCount` of the producer for topic " + + destination.getName() + " is " + + producerProperties.getPartitionCount() + + ", smaller than the actual partition count of " + + partitions.size() + + " for the topic. The larger number will be used instead."); + } + List interceptors = ((InterceptableChannel) channel) + .getInterceptors(); + interceptors.forEach((interceptor) -> { + if (interceptor instanceof PartitioningInterceptor) { + ((PartitioningInterceptor) interceptor) + .setPartitionCount(partitions.size()); + } + else if (interceptor instanceof DefaultPartitioningInterceptor) { + ((DefaultPartitioningInterceptor) interceptor) + .setPartitionCount(partitions.size()); + } + }); + } + + KafkaTemplate kafkaTemplate = new KafkaTemplate<>(producerFB); + if (this.producerListener != null) { + kafkaTemplate.setProducerListener(this.producerListener); + } + if (transMan != null) { + kafkaTemplate.setTransactionIdPrefix(configurationProperties.getTransaction().getTransactionIdPrefix()); + } + final boolean allowNonTransactional = producerProperties.getExtension().isAllowNonTransactional(); + if (allowNonTransactional) { + kafkaTemplate.setAllowNonTransactional(allowNonTransactional); + } + ProducerConfigurationMessageHandler handler = new ProducerConfigurationMessageHandler( + kafkaTemplate, destination.getName(), producerProperties, producerFB); + if (errorChannel != null) { + handler.setSendFailureChannel(errorChannel); + } + if (StringUtils.hasText(producerProperties.getExtension().getRecordMetadataChannel())) { + handler.setSendSuccessChannelName(producerProperties.getExtension().getRecordMetadataChannel()); + } + AbstractApplicationContext applicationContext = getApplicationContext(); + handler.setApplicationContext(applicationContext); + + KafkaHeaderMapper mapper = null; + if (this.configurationProperties.getHeaderMapperBeanName() != null) { + mapper = applicationContext.getBean( + this.configurationProperties.getHeaderMapperBeanName(), + KafkaHeaderMapper.class); + } + if (mapper == null) { + //First, try to see if there is a bean named headerMapper registered by other frameworks using the binder (for e.g. spring cloud sleuth) + try { + mapper = applicationContext.getBean("kafkaBinderHeaderMapper", KafkaHeaderMapper.class); + } + catch (BeansException be) { + // Pass through + } + } + + /* + * Even if the user configures a bean, we must not use it if the header mode is + * not the default (headers); setting the mapper to null disables populating + * headers in the message handler. + */ + if (producerProperties.getHeaderMode() != null + && !HeaderMode.headers.equals(producerProperties.getHeaderMode())) { + mapper = null; + } + else if (mapper == null) { + String[] headerPatterns = producerProperties.getExtension().getHeaderPatterns(); + if (headerPatterns != null && headerPatterns.length > 0) { + mapper = new BinderHeaderMapper( + BinderHeaderMapper.addNeverHeaderPatterns(Arrays.asList(headerPatterns))); + } + else { + mapper = new BinderHeaderMapper(); + } + } + else { + KafkaHeaderMapper userHeaderMapper = mapper; + mapper = new KafkaHeaderMapper() { + + @Override + public void toHeaders(Headers source, Map target) { + userHeaderMapper.toHeaders(source, target); + } + + @Override + public void fromHeaders(MessageHeaders headers, Headers target) { + userHeaderMapper.fromHeaders(headers, target); + BinderHeaderMapper.removeNeverHeaders(target); + } + }; + + } + handler.setHeaderMapper(mapper); + return handler; + } + + + @Override + protected void postProcessOutputChannel(MessageChannel outputChannel, + ExtendedProducerProperties producerProperties) { + + if (expressionInterceptorNeeded(producerProperties)) { + ((AbstractMessageChannel) outputChannel).addInterceptor(0, new KafkaExpressionEvaluatingInterceptor( + producerProperties.getExtension().getMessageKeyExpression(), getEvaluationContext())); + } + } + + private boolean expressionInterceptorNeeded( + ExtendedProducerProperties producerProperties) { + if (producerProperties.isUseNativeEncoding()) { + return false; // payload will be intact when it reaches the adapter + } + else { + Expression messageKeyExpression = producerProperties.getExtension().getMessageKeyExpression(); + return messageKeyExpression != null + && interceptorNeededPattern.matcher(messageKeyExpression.getExpressionString()).find(); + } + } + + protected DefaultKafkaProducerFactory getProducerFactory( + String transactionIdPrefix, + ExtendedProducerProperties producerProperties, String beanName, String destination) { + Map props = new HashMap<>(); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + ByteArraySerializer.class); + props.put(ProducerConfig.ACKS_CONFIG, + String.valueOf(this.configurationProperties.getRequiredAcks())); + Map mergedConfig = this.configurationProperties + .mergedProducerConfiguration(); + if (!ObjectUtils.isEmpty(mergedConfig)) { + props.putAll(mergedConfig); + } + if (ObjectUtils.isEmpty(props.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG))) { + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + this.configurationProperties.getKafkaConnectionString()); + } + final KafkaProducerProperties kafkaProducerProperties = producerProperties.getExtension(); + if (ObjectUtils.isEmpty(props.get(ProducerConfig.BATCH_SIZE_CONFIG))) { + props.put(ProducerConfig.BATCH_SIZE_CONFIG, + String.valueOf(kafkaProducerProperties.getBufferSize())); + } + if (ObjectUtils.isEmpty(props.get(ProducerConfig.LINGER_MS_CONFIG))) { + props.put(ProducerConfig.LINGER_MS_CONFIG, + String.valueOf(kafkaProducerProperties.getBatchTimeout())); + } + if (ObjectUtils.isEmpty(props.get(ProducerConfig.COMPRESSION_TYPE_CONFIG))) { + props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, + kafkaProducerProperties.getCompressionType().toString()); + } + Map configs = producerProperties.getExtension().getConfiguration(); + Assert.state(!configs.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG), + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " cannot be overridden at the binding level; " + + "use multiple binders instead"); + if (!ObjectUtils.isEmpty(configs)) { + props.putAll(configs); + } + if (!ObjectUtils.isEmpty(kafkaProducerProperties.getConfiguration())) { + props.putAll(kafkaProducerProperties.getConfiguration()); + } + if (this.producerConfigCustomizer != null) { + this.producerConfigCustomizer.configure(props, bindingNameHolder.get(), destination); + bindingNameHolder.remove(); + } + DefaultKafkaProducerFactory producerFactory = new DefaultKafkaProducerFactory<>( + props); + if (transactionIdPrefix != null) { + producerFactory.setTransactionIdPrefix(transactionIdPrefix); + } + if (kafkaProducerProperties.getCloseTimeout() > 0) { + producerFactory.setPhysicalCloseTimeout(kafkaProducerProperties.getCloseTimeout()); + } + producerFactory.setBeanName(beanName); + if (this.clientFactoryCustomizer != null) { + this.clientFactoryCustomizer.configure(producerFactory); + } + return producerFactory; + } + + @Override + protected boolean useNativeEncoding( + ExtendedProducerProperties producerProperties) { + if (transactionManager(producerProperties.getExtension().getTransactionManager()) != null) { + return this.configurationProperties.getTransaction().getProducer() + .isUseNativeEncoding(); + } + return super.useNativeEncoding(producerProperties); + } + + @Override + @SuppressWarnings("unchecked") + protected MessageProducer createConsumerEndpoint( + final ConsumerDestination destination, final String group, + final ExtendedConsumerProperties extendedConsumerProperties) { + + boolean anonymous = !StringUtils.hasText(group); + Assert.isTrue( + !anonymous || !extendedConsumerProperties.getExtension().isEnableDlq(), + "DLQ support is not available for anonymous subscriptions"); + String consumerGroup = anonymous ? "anonymous." + UUID.randomUUID().toString() + : group; + final ConsumerFactory consumerFactory = createKafkaConsumerFactory( + anonymous, consumerGroup, extendedConsumerProperties, destination.getName() + ".consumer", destination.getName()); + int partitionCount = extendedConsumerProperties.getInstanceCount() + * extendedConsumerProperties.getConcurrency(); + + Collection listenedPartitions = new ArrayList<>(); + + boolean usingPatterns = extendedConsumerProperties.getExtension() + .isDestinationIsPattern(); + Assert.isTrue(!usingPatterns || !extendedConsumerProperties.isMultiplex(), + "Cannot use a pattern with multiplexed destinations; " + + "use the regex pattern to specify multiple topics instead"); + boolean groupManagement = extendedConsumerProperties.getExtension() + .isAutoRebalanceEnabled(); + if (!extendedConsumerProperties.isMultiplex()) { + listenedPartitions.addAll(processTopic(consumerGroup, + extendedConsumerProperties, consumerFactory, partitionCount, + usingPatterns, groupManagement, destination.getName())); + } + else { + for (String name : StringUtils + .commaDelimitedListToStringArray(destination.getName())) { + listenedPartitions.addAll(processTopic(consumerGroup, + extendedConsumerProperties, consumerFactory, partitionCount, + usingPatterns, groupManagement, name.trim())); + } + } + + String[] topics = extendedConsumerProperties.isMultiplex() + ? StringUtils.commaDelimitedListToStringArray(destination.getName()) + : new String[] { destination.getName() }; + for (int i = 0; i < topics.length; i++) { + topics[i] = topics[i].trim(); + } + if (!usingPatterns && !groupManagement) { + Assert.isTrue(!CollectionUtils.isEmpty(listenedPartitions), + "A list of partitions must be provided"); + } + final TopicPartitionOffset[] topicPartitionOffsets = groupManagement + ? null + : getTopicPartitionOffsets(listenedPartitions, extendedConsumerProperties, consumerFactory); + final ContainerProperties containerProperties = anonymous + || groupManagement + ? usingPatterns + ? new ContainerProperties(Pattern.compile(topics[0])) + : new ContainerProperties(topics) + : new ContainerProperties(topicPartitionOffsets); + KafkaAwareTransactionManager transMan = transactionManager( + extendedConsumerProperties.getExtension().getTransactionManager()); + if (transMan != null) { + containerProperties.setTransactionManager(transMan); + } + if (this.rebalanceListener != null) { + setupRebalanceListener(extendedConsumerProperties, containerProperties); + } + containerProperties.setIdleEventInterval( + extendedConsumerProperties.getExtension().getIdleEventInterval()); + int concurrency = usingPatterns ? extendedConsumerProperties.getConcurrency() + : Math.min(extendedConsumerProperties.getConcurrency(), + listenedPartitions.size()); + // in the event that auto rebalance is enabled, but no listened partitions are found + // we want to make sure that concurrency is a non-zero value. + if (groupManagement && listenedPartitions.isEmpty()) { + concurrency = extendedConsumerProperties.getConcurrency(); + } + resetOffsetsForAutoRebalance(extendedConsumerProperties, consumerFactory, containerProperties); + containerProperties.setAuthorizationExceptionRetryInterval(this.configurationProperties.getAuthorizationExceptionRetryInterval()); + @SuppressWarnings("rawtypes") + final ConcurrentMessageListenerContainer messageListenerContainer = new ConcurrentMessageListenerContainer( + consumerFactory, containerProperties) { + + @Override + public void stop(Runnable callback) { + super.stop(callback); + } + + }; + + this.kafkaMessageListenerContainers.add(messageListenerContainer); + messageListenerContainer.setConcurrency(concurrency); + // these won't be needed if the container is made a bean + AbstractApplicationContext applicationContext = getApplicationContext(); + messageListenerContainer.setApplicationContext(applicationContext); + if (getApplicationEventPublisher() != null) { + messageListenerContainer + .setApplicationEventPublisher(getApplicationEventPublisher()); + } + else if (applicationContext != null) { + messageListenerContainer + .setApplicationEventPublisher(applicationContext); + } + messageListenerContainer.setBeanName(destination + ".container"); + // end of these won't be needed... + ContainerProperties.AckMode ackMode = extendedConsumerProperties.getExtension().getAckMode(); + if (ackMode == null) { + if (extendedConsumerProperties.getExtension().isAckEachRecord()) { + ackMode = ContainerProperties.AckMode.RECORD; + } + else { + if (!extendedConsumerProperties.getExtension().isAutoCommitOffset()) { + messageListenerContainer.getContainerProperties() + .setAckMode(ContainerProperties.AckMode.MANUAL); + } + } + } + if (ackMode != null) { + if ((extendedConsumerProperties.isBatchMode() && ackMode != ContainerProperties.AckMode.RECORD) || + !extendedConsumerProperties.isBatchMode()) { + messageListenerContainer.getContainerProperties() + .setAckMode(ackMode); + } + } + + if (this.logger.isDebugEnabled()) { + this.logger.debug("Listened partitions: " + + StringUtils.collectionToCommaDelimitedString(listenedPartitions)); + } + final KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = + new KafkaMessageDrivenChannelAdapter<>(messageListenerContainer, + extendedConsumerProperties.isBatchMode() ? ListenerMode.batch : ListenerMode.record); + MessageConverter messageConverter = getMessageConverter(extendedConsumerProperties); + kafkaMessageDrivenChannelAdapter.setMessageConverter(messageConverter); + kafkaMessageDrivenChannelAdapter.setBeanFactory(getBeanFactory()); + kafkaMessageDrivenChannelAdapter.setApplicationContext(applicationContext); + ErrorInfrastructure errorInfrastructure = registerErrorInfrastructure(destination, + consumerGroup, extendedConsumerProperties); + + ListenerContainerCustomizer customizer = getContainerCustomizer(); + + if (!extendedConsumerProperties.isBatchMode() + && extendedConsumerProperties.getMaxAttempts() > 1 + && transMan == null) { + if (!(customizer instanceof ListenerContainerWithDlqAndRetryCustomizer) + || ((ListenerContainerWithDlqAndRetryCustomizer) customizer) + .retryAndDlqInBinding(destination.getName(), group)) { + kafkaMessageDrivenChannelAdapter + .setRetryTemplate(buildRetryTemplate(extendedConsumerProperties)); + kafkaMessageDrivenChannelAdapter + .setRecoveryCallback(errorInfrastructure.getRecoverer()); + } + if (!extendedConsumerProperties.getExtension().isEnableDlq()) { + messageListenerContainer.setCommonErrorHandler(new DefaultErrorHandler(new FixedBackOff(0L, 0L))); + } + } + else if (!extendedConsumerProperties.isBatchMode() && transMan != null) { + messageListenerContainer.setAfterRollbackProcessor(new DefaultAfterRollbackProcessor<>( + (record, exception) -> { + MessagingException payload = + new MessagingException(((RecordMessageConverter) messageConverter) + .toMessage(record, null, null, null), + "Transaction rollback limit exceeded", exception); + try { + errorInfrastructure.getErrorChannel() + .send(new ErrorMessage( + payload, + Collections.singletonMap(IntegrationMessageHeaderAccessor.SOURCE_DATA, + record))); + } + catch (Exception e) { + /* + * When there is no DLQ, the FinalRethrowingErrorMessageHandler will re-throw + * the payload; that will subvert the recovery and cause a re-seek of the failed + * record, so we ignore that here. + */ + if (!e.equals(payload)) { + throw e; + } + } + }, createBackOff(extendedConsumerProperties), + new KafkaTemplate<>(transMan.getProducerFactory()), + extendedConsumerProperties.getExtension().isTxCommitRecovered())); + } + else { + kafkaMessageDrivenChannelAdapter.setErrorChannel(errorInfrastructure.getErrorChannel()); + } + final String commonErrorHandlerBeanName = extendedConsumerProperties.getExtension().getCommonErrorHandlerBeanName(); + if (StringUtils.hasText(commonErrorHandlerBeanName)) { + final CommonErrorHandler commonErrorHandler = getApplicationContext().getBean(commonErrorHandlerBeanName, + CommonErrorHandler.class); + messageListenerContainer.setCommonErrorHandler(commonErrorHandler); + } + if (customizer instanceof ListenerContainerWithDlqAndRetryCustomizer) { + + BiFunction, Exception, TopicPartition> destinationResolver = createDestResolver( + extendedConsumerProperties.getExtension()); + BackOff createBackOff = extendedConsumerProperties.getMaxAttempts() > 1 + ? createBackOff(extendedConsumerProperties) + : null; + ((ListenerContainerWithDlqAndRetryCustomizer) customizer) + .configure(messageListenerContainer, destination.getName(), consumerGroup, destinationResolver, + createBackOff); + } + else { + ((ListenerContainerCustomizer) customizer) + .configure(messageListenerContainer, destination.getName(), consumerGroup); + } + this.ackModeInfo.put(destination, messageListenerContainer.getContainerProperties().getAckMode()); + return kafkaMessageDrivenChannelAdapter; + } + + private BiFunction, Exception, TopicPartition> createDestResolver( + KafkaConsumerProperties extension) { + + Integer dlqPartitions = extension.getDlqPartitions(); + if (extension.isEnableDlq()) { + return (rec, ex) -> dlqPartitions == null || dlqPartitions > 1 + ? new TopicPartition(extension.getDlqName(), rec.partition()) + : new TopicPartition(extension.getDlqName(), 0); + } + else { + return null; + } + } + + /** + * Configure a {@link BackOff} for the after rollback processor, based on the consumer + * retry properties. If retry is disabled, return a {@link BackOff} that disables + * retry. Otherwise use an {@link ExponentialBackOffWithMaxRetries}. + * @param extendedConsumerProperties the properties. + * @return the backoff. + */ + private BackOff createBackOff( + final ExtendedConsumerProperties extendedConsumerProperties) { + + int maxAttempts = extendedConsumerProperties.getMaxAttempts(); + if (maxAttempts < 2) { + return new FixedBackOff(0L, 0L); + } + int initialInterval = extendedConsumerProperties.getBackOffInitialInterval(); + int maxInterval = extendedConsumerProperties.getBackOffMaxInterval(); + ExponentialBackOff backOff = new ExponentialBackOffWithMaxRetries(maxAttempts - 1); + backOff.setInitialInterval(initialInterval); + backOff.setMaxInterval(maxInterval); + return backOff; + } + + public void setupRebalanceListener( + final ExtendedConsumerProperties extendedConsumerProperties, + final ContainerProperties containerProperties) { + + Assert.isTrue(!extendedConsumerProperties.getExtension().isResetOffsets(), + "'resetOffsets' cannot be set when a KafkaBindingRebalanceListener is provided"); + final String bindingName = bindingNameHolder.get(); + bindingNameHolder.remove(); + Assert.notNull(bindingName, "'bindingName' cannot be null"); + final KafkaBindingRebalanceListener userRebalanceListener = this.rebalanceListener; + containerProperties + .setConsumerRebalanceListener(new ConsumerAwareRebalanceListener() { + + private final ThreadLocal initialAssignment = new ThreadLocal<>(); + + @Override + public void onPartitionsRevokedBeforeCommit(Consumer consumer, + Collection partitions) { + + userRebalanceListener.onPartitionsRevokedBeforeCommit(bindingName, + consumer, partitions); + } + + @Override + public void onPartitionsRevokedAfterCommit(Consumer consumer, + Collection partitions) { + + userRebalanceListener.onPartitionsRevokedAfterCommit(bindingName, + consumer, partitions); + } + + @Override + public void onPartitionsAssigned(Consumer consumer, + Collection partitions) { + try { + Boolean initial = this.initialAssignment.get(); + if (initial == null) { + initial = Boolean.TRUE; + } + userRebalanceListener.onPartitionsAssigned(bindingName, + consumer, partitions, initial); + } + finally { + this.initialAssignment.set(Boolean.FALSE); + } + } + + }); + } + + public Collection processTopic(final String group, + final ExtendedConsumerProperties extendedConsumerProperties, + final ConsumerFactory consumerFactory, int partitionCount, + boolean usingPatterns, boolean groupManagement, String topic) { + Collection listenedPartitions; + Collection allPartitions = usingPatterns ? Collections.emptyList() + : getPartitionInfo(topic, extendedConsumerProperties, consumerFactory, + partitionCount); + + if (groupManagement || extendedConsumerProperties.getInstanceCount() == 1) { + listenedPartitions = allPartitions; + } + else { + listenedPartitions = new ArrayList<>(); + for (PartitionInfo partition : allPartitions) { + // divide partitions across modules + if ((partition.partition() % extendedConsumerProperties + .getInstanceCount()) == extendedConsumerProperties + .getInstanceIndex()) { + listenedPartitions.add(partition); + } + } + } + this.topicsInUse.put(topic, + new TopicInformation(group, listenedPartitions, usingPatterns)); + return listenedPartitions; + } + + /* + * Reset the offsets if needed; may update the offsets in in the container's + * topicPartitionInitialOffsets. + */ + private void resetOffsetsForAutoRebalance( + final ExtendedConsumerProperties extendedConsumerProperties, + final ConsumerFactory consumerFactory, final ContainerProperties containerProperties) { + + final Object resetTo = checkReset(extendedConsumerProperties.getExtension().isResetOffsets(), + consumerFactory.getConfigurationProperties() + .get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)); + if (resetTo != null) { + Set sought = ConcurrentHashMap.newKeySet(); + containerProperties.setConsumerRebalanceListener(new ConsumerAwareRebalanceListener() { + + @Override + public void onPartitionsRevokedBeforeCommit( + Consumer consumer, Collection tps) { + + if (logger.isInfoEnabled()) { + logger.info("Partitions revoked: " + tps); + } + } + + @Override + public void onPartitionsRevokedAfterCommit( + Consumer consumer, Collection tps) { + // no op + } + + @Override + public void onPartitionsAssigned(Consumer consumer, Collection tps) { + if (logger.isInfoEnabled()) { + logger.info("Partitions assigned: " + tps); + } + List toSeek = tps.stream() + .filter(tp -> { + boolean shouldSeek = !sought.contains(tp); + sought.add(tp); + return shouldSeek; + }) + .collect(Collectors.toList()); + if (toSeek.size() > 0) { + if ("earliest".equals(resetTo)) { + consumer.seekToBeginning(toSeek); + } + else if ("latest".equals(resetTo)) { + consumer.seekToEnd(toSeek); + } + } + } + }); + } + } + + private Object checkReset(boolean resetOffsets, final Object resetTo) { + if (!resetOffsets) { + return null; + } + else if (!"earliest".equals(resetTo) && !"latest".equals(resetTo)) { + logger.warn("no (or unknown) " + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG + + " property cannot reset"); + return null; + } + else { + return resetTo; + } + } + + @Override + protected PolledConsumerResources createPolledConsumerResources(String name, + String group, ConsumerDestination destination, + ExtendedConsumerProperties extendedConsumerProperties) { + + boolean anonymous = !StringUtils.hasText(group); + final KafkaConsumerProperties extension = extendedConsumerProperties.getExtension(); + Assert.isTrue(!anonymous || !extension.isEnableDlq(), + "DLQ support is not available for anonymous subscriptions"); + String consumerGroup = anonymous ? "anonymous." + UUID.randomUUID().toString() + : group; + final ConsumerFactory consumerFactory = createKafkaConsumerFactory( + anonymous, consumerGroup, extendedConsumerProperties, destination.getName() + ".polled.consumer", + destination.getName()); + String[] topics = extendedConsumerProperties.isMultiplex() + ? StringUtils.commaDelimitedListToStringArray(destination.getName()) + : new String[] { destination.getName() }; + for (int i = 0; i < topics.length; i++) { + topics[i] = topics[i].trim(); + } + final ConsumerProperties consumerProperties = new ConsumerProperties(topics); + + String clientId = name; + if (extension.getConfiguration() + .containsKey(ConsumerConfig.CLIENT_ID_CONFIG)) { + clientId = extension.getConfiguration() + .get(ConsumerConfig.CLIENT_ID_CONFIG); + } + + consumerProperties.setClientId(clientId); + + consumerProperties.setConsumerRebalanceListener(new ConsumerRebalanceListener() { + + @Override + public void onPartitionsRevoked(Collection partitions) { + KafkaMessageChannelBinder.this.logger.info("Revoked: " + partitions); + } + + @Override + public void onPartitionsAssigned(Collection partitions) { + KafkaMessageChannelBinder.this.logger.info("Assigned: " + partitions); + } + + }); + + consumerProperties.setPollTimeout(extension.getPollTimeout()); + + KafkaMessageSource source = new KafkaMessageSource<>(consumerFactory, + consumerProperties); + MessageConverter messageConverter = getMessageConverter(extendedConsumerProperties); + Assert.isInstanceOf(RecordMessageConverter.class, messageConverter, + "'messageConverter' must be a 'RecordMessageConverter' for polled consumers"); + source.setMessageConverter((RecordMessageConverter) messageConverter); + source.setRawMessageHeader(extension.isEnableDlq()); + + if (!extendedConsumerProperties.isMultiplex()) { + // I copied this from the regular consumer - it looks bogus to me - includes + // all partitions + // not just the ones this binding is listening to; doesn't seem right for a + // health check. + Collection partitionInfos = getPartitionInfo( + destination.getName(), extendedConsumerProperties, consumerFactory, -1); + this.topicsInUse.put(destination.getName(), + new TopicInformation(consumerGroup, partitionInfos, false)); + } + else { + for (int i = 0; i < topics.length; i++) { + Collection partitionInfos = getPartitionInfo(topics[i], + extendedConsumerProperties, consumerFactory, -1); + this.topicsInUse.put(topics[i], + new TopicInformation(consumerGroup, partitionInfos, false)); + } + } + + getMessageSourceCustomizer().configure(source, destination.getName(), group); + return new PolledConsumerResources(source, registerErrorInfrastructure( + destination, group, extendedConsumerProperties, true)); + } + + @Override + protected void postProcessPollableSource(DefaultPollableMessageSource bindingTarget) { + bindingTarget.setAttributesProvider((accessor, message) -> { + Object rawMessage = message.getHeaders().get(KafkaHeaders.RAW_DATA); + if (rawMessage != null) { + accessor.setAttribute(KafkaHeaders.RAW_DATA, rawMessage); + } + }); + } + + private MessageConverter getMessageConverter( + final ExtendedConsumerProperties extendedConsumerProperties) { + + MessageConverter messageConverter; + if (extendedConsumerProperties.getExtension().getConverterBeanName() == null) { + MessagingMessageConverter mmc = new MessagingMessageConverter(); + StandardHeaders standardHeaders = extendedConsumerProperties.getExtension() + .getStandardHeaders(); + mmc.setGenerateMessageId(StandardHeaders.id.equals(standardHeaders) + || StandardHeaders.both.equals(standardHeaders)); + mmc.setGenerateTimestamp( + StandardHeaders.timestamp.equals(standardHeaders) + || StandardHeaders.both.equals(standardHeaders)); + messageConverter = mmc; + } + else { + try { + messageConverter = getApplicationContext().getBean( + extendedConsumerProperties.getExtension().getConverterBeanName(), + MessageConverter.class); + } + catch (NoSuchBeanDefinitionException ex) { + throw new IllegalStateException( + "Converter bean not present in application context", ex); + } + } + if (messageConverter instanceof MessagingMessageConverter) { + ((MessagingMessageConverter) messageConverter).setHeaderMapper(getHeaderMapper(extendedConsumerProperties)); + } + return messageConverter; + } + + private KafkaHeaderMapper getHeaderMapper( + final ExtendedConsumerProperties extendedConsumerProperties) { + KafkaHeaderMapper mapper = null; + if (this.configurationProperties.getHeaderMapperBeanName() != null) { + mapper = getApplicationContext().getBean( + this.configurationProperties.getHeaderMapperBeanName(), + KafkaHeaderMapper.class); + } + if (mapper == null) { + //First, try to see if there is a bean named headerMapper registered by other frameworks using the binder (for e.g. spring cloud sleuth) + try { + mapper = getApplicationContext().getBean("kafkaBinderHeaderMapper", KafkaHeaderMapper.class); + } + catch (BeansException be) { + BinderHeaderMapper headerMapper = new BinderHeaderMapper() { + + @Override + public void toHeaders(Headers source, Map headers) { + super.toHeaders(source, headers); + if (headers.size() > 0) { + headers.put(BinderHeaders.NATIVE_HEADERS_PRESENT, Boolean.TRUE); + } + } + + }; + String[] trustedPackages = extendedConsumerProperties.getExtension() + .getTrustedPackages(); + if (!StringUtils.isEmpty(trustedPackages)) { + headerMapper.addTrustedPackages(trustedPackages); + } + mapper = headerMapper; + } + } + return mapper; + } + + private Collection getPartitionInfo(String topic, + final ExtendedConsumerProperties extendedConsumerProperties, + final ConsumerFactory consumerFactory, int partitionCount) { + return provisioningProvider.getPartitionsForTopic(partitionCount, + extendedConsumerProperties.getExtension().isAutoRebalanceEnabled(), + () -> { + try (Consumer consumer = consumerFactory.createConsumer()) { + return consumer.partitionsFor(topic); + } + }, topic); + } + + @Override + protected ErrorMessageStrategy getErrorMessageStrategy() { + return new RawRecordHeaderErrorMessageStrategy(); + } + + @SuppressWarnings("unchecked") + @Override + protected MessageHandler getErrorMessageHandler(final ConsumerDestination destination, + final String group, + final ExtendedConsumerProperties properties) { + + KafkaConsumerProperties kafkaConsumerProperties = properties.getExtension(); + if (kafkaConsumerProperties.isEnableDlq()) { + KafkaProducerProperties dlqProducerProperties = kafkaConsumerProperties + .getDlqProducerProperties(); + KafkaAwareTransactionManager transMan = transactionManager( + properties.getExtension().getTransactionManager()); + ProducerFactory producerFactory = transMan != null + ? transMan.getProducerFactory() + : getProducerFactory(null, + new ExtendedProducerProperties<>(dlqProducerProperties), + destination.getName() + ".dlq.producer", destination.getName()); + final KafkaTemplate kafkaTemplate = new KafkaTemplate<>( + producerFactory); + + Object timeout = producerFactory.getConfigurationProperties().get(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG); + Long sendTimeout = null; + if (timeout instanceof Number) { + sendTimeout = ((Number) timeout).longValue() + 2000L; + } + else if (timeout instanceof String) { + sendTimeout = Long.parseLong((String) timeout) + 2000L; + } + if (timeout == null) { + sendTimeout = ((Integer) ProducerConfig.configDef() + .defaultValues() + .get(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG)).longValue() + 2000L; + } + @SuppressWarnings("rawtypes") + DlqSender dlqSender = new DlqSender(kafkaTemplate, sendTimeout); + + return (message) -> { + + ConsumerRecord record = StaticMessageHeaderAccessor.getSourceData(message); + + if (properties.isUseNativeDecoding()) { + if (record != null) { + // Give the binder configuration the least preference. + Map configuration = this.configurationProperties.getConfiguration(); + // Then give any producer specific properties specified on the binder. + configuration.putAll(this.configurationProperties.getProducerProperties()); + Map configs = transMan == null + ? dlqProducerProperties.getConfiguration() + : this.configurationProperties.getTransaction() + .getProducer().getConfiguration(); + Assert.state(!configs.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG), + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " cannot be overridden at the binding level; " + + "use multiple binders instead"); + // Finally merge with dlq producer properties or the transaction producer properties. + configuration.putAll(configs); + if (record.key() != null + && !record.key().getClass().isInstance(byte[].class)) { + ensureDlqMessageCanBeProperlySerialized(configuration, + (Map config) -> !config + .containsKey("key.serializer"), + "Key"); + } + if (record.value() != null + && !record.value().getClass().isInstance(byte[].class)) { + ensureDlqMessageCanBeProperlySerialized(configuration, + (Map config) -> !config + .containsKey("value.serializer"), + "Payload"); + } + } + } + + if (record == null) { + this.logger.error("No raw record; cannot send to DLQ: " + message); + return; + } + Headers kafkaHeaders = new RecordHeaders(record.headers().toArray()); + AtomicReference> recordToSend = new AtomicReference<>( + record); + Throwable throwable = null; + if (message.getPayload() instanceof Throwable) { + + throwable = (Throwable) message.getPayload(); + + HeaderMode headerMode = properties.getHeaderMode(); + + if (headerMode == null || HeaderMode.headers.equals(headerMode)) { + + kafkaHeaders.add(new RecordHeader(X_ORIGINAL_TOPIC, + record.topic().getBytes(StandardCharsets.UTF_8))); + kafkaHeaders.add(new RecordHeader(X_ORIGINAL_PARTITION, + ByteBuffer.allocate(Integer.BYTES) + .putInt(record.partition()).array())); + kafkaHeaders.add(new RecordHeader(X_ORIGINAL_OFFSET, ByteBuffer + .allocate(Long.BYTES).putLong(record.offset()).array())); + kafkaHeaders.add(new RecordHeader(X_ORIGINAL_TIMESTAMP, + ByteBuffer.allocate(Long.BYTES) + .putLong(record.timestamp()).array())); + kafkaHeaders.add(new RecordHeader(X_ORIGINAL_TIMESTAMP_TYPE, + record.timestampType().toString() + .getBytes(StandardCharsets.UTF_8))); + kafkaHeaders.add(new RecordHeader(X_EXCEPTION_FQCN, throwable + .getClass().getName().getBytes(StandardCharsets.UTF_8))); + String exceptionMessage = throwable.getMessage(); + if (exceptionMessage != null) { + kafkaHeaders.add(new RecordHeader(X_EXCEPTION_MESSAGE, + exceptionMessage.getBytes(StandardCharsets.UTF_8))); + } + kafkaHeaders.add(new RecordHeader(X_EXCEPTION_STACKTRACE, + getStackTraceAsString(throwable) + .getBytes(StandardCharsets.UTF_8))); + } + else if (HeaderMode.embeddedHeaders.equals(headerMode)) { + try { + MessageValues messageValues = EmbeddedHeaderUtils + .extractHeaders(MessageBuilder + .withPayload((byte[]) record.value()).build(), + false); + messageValues.put(X_ORIGINAL_TOPIC, record.topic()); + messageValues.put(X_ORIGINAL_PARTITION, record.partition()); + messageValues.put(X_ORIGINAL_OFFSET, record.offset()); + messageValues.put(X_ORIGINAL_TIMESTAMP, record.timestamp()); + messageValues.put(X_ORIGINAL_TIMESTAMP_TYPE, + record.timestampType().toString()); + messageValues.put(X_EXCEPTION_FQCN, + throwable.getClass().getName()); + messageValues.put(X_EXCEPTION_MESSAGE, + throwable.getMessage()); + messageValues.put(X_EXCEPTION_STACKTRACE, + getStackTraceAsString(throwable)); + + final String[] headersToEmbed = new ArrayList<>( + messageValues.keySet()).toArray( + new String[messageValues.keySet().size()]); + byte[] payload = EmbeddedHeaderUtils.embedHeaders( + messageValues, + EmbeddedHeaderUtils.headersToEmbed(headersToEmbed)); + recordToSend.set(new ConsumerRecord( + record.topic(), record.partition(), record.offset(), + record.key(), payload)); + } + catch (Exception ex) { + throw new RuntimeException(ex); + } + } + } + + MessageHeaders headers; + if (message instanceof ErrorMessage) { + final ErrorMessage errorMessage = (ErrorMessage) message; + final Message originalMessage = errorMessage.getOriginalMessage(); + if (originalMessage != null) { + headers = originalMessage.getHeaders(); + } + else { + headers = message.getHeaders(); + } + } + else { + headers = message.getHeaders(); + } + String dlqName = this.dlqDestinationResolver != null ? + this.dlqDestinationResolver.apply(recordToSend.get(), new Exception(throwable)) : StringUtils.hasText(kafkaConsumerProperties.getDlqName()) + ? kafkaConsumerProperties.getDlqName() + : "error." + record.topic() + "." + group; + if (this.transactionTemplate != null) { + Throwable throwable2 = throwable; + this.transactionTemplate.executeWithoutResult(status -> { + dlqSender.sendToDlq(recordToSend.get(), kafkaHeaders, dlqName, group, throwable2, + determinDlqPartitionFunction(properties.getExtension().getDlqPartitions()), + headers, this.ackModeInfo.get(destination)); + }); + } + else { + dlqSender.sendToDlq(recordToSend.get(), kafkaHeaders, dlqName, group, throwable, + determinDlqPartitionFunction(properties.getExtension().getDlqPartitions()), headers, this.ackModeInfo.get(destination)); + } + }; + } + return null; + } + + @SuppressWarnings("unchecked") + @Nullable + private KafkaAwareTransactionManager transactionManager(@Nullable String beanName) { + if (StringUtils.hasText(beanName)) { + return getApplicationContext().getBean(beanName, KafkaAwareTransactionManager.class); + } + return this.transactionManager; + } + + private DlqPartitionFunction determinDlqPartitionFunction(Integer dlqPartitions) { + if (this.dlqPartitionFunction != null) { + return this.dlqPartitionFunction; + } + else { + return DlqPartitionFunction.determineFallbackFunction(dlqPartitions, this.logger); + } + } + + @Override + protected MessageHandler getPolledConsumerErrorMessageHandler( + ConsumerDestination destination, String group, + ExtendedConsumerProperties properties) { + if (properties.getExtension().isEnableDlq()) { + return getErrorMessageHandler(destination, group, properties); + } + final MessageHandler superHandler = super.getErrorMessageHandler(destination, + group, properties); + return (message) -> { + ConsumerRecord record = (ConsumerRecord) message.getHeaders() + .get(KafkaHeaders.RAW_DATA); + if (!(message instanceof ErrorMessage)) { + logger.error("Expected an ErrorMessage, not a " + + message.getClass().toString() + " for: " + message); + } + else if (record == null) { + if (superHandler != null) { + superHandler.handleMessage(message); + } + } + else { + if (message.getPayload() instanceof MessagingException) { + AcknowledgmentCallback ack = StaticMessageHeaderAccessor + .getAcknowledgmentCallback( + ((MessagingException) message.getPayload()) + .getFailedMessage()); + if (ack != null) { + if (isAutoCommitOnError(properties)) { + ack.acknowledge(AcknowledgmentCallback.Status.REJECT); + } + else { + ack.acknowledge(AcknowledgmentCallback.Status.REQUEUE); + } + } + } + } + }; + } + + private static void ensureDlqMessageCanBeProperlySerialized( + Map configuration, + Predicate> configPredicate, String dataType) { + if (CollectionUtils.isEmpty(configuration) + || configPredicate.test(configuration)) { + throw new IllegalArgumentException("Native decoding is used on the consumer. " + + dataType + + " is not byte[] and no serializer is set on the DLQ producer."); + } + } + + protected ConsumerFactory createKafkaConsumerFactory(boolean anonymous, + String consumerGroup, ExtendedConsumerProperties consumerProperties, + String beanName, String destination) { + Map props = new HashMap<>(); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + ByteArrayDeserializer.class); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + ByteArrayDeserializer.class); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); + props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 100); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, + anonymous ? "latest" : "earliest"); + props.put(ConsumerConfig.GROUP_ID_CONFIG, consumerGroup); + + Map mergedConfig = this.configurationProperties + .mergedConsumerConfiguration(); + if (!ObjectUtils.isEmpty(mergedConfig)) { + props.putAll(mergedConfig); + } + if (ObjectUtils.isEmpty(props.get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG))) { + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + this.configurationProperties.getKafkaConnectionString()); + } + Map config = consumerProperties.getExtension().getConfiguration(); + if (!ObjectUtils.isEmpty(config)) { + Assert.state(!config.containsKey(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG), + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG + " cannot be overridden at the binding level; " + + "use multiple binders instead"); + props.putAll(config); + } + if (!ObjectUtils.isEmpty(consumerProperties.getExtension().getStartOffset())) { + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, + consumerProperties.getExtension().getStartOffset().name()); + } + + if (this.consumerConfigCustomizer != null) { + this.consumerConfigCustomizer.configure(props, bindingNameHolder.get(), destination); + } + DefaultKafkaConsumerFactory factory = new DefaultKafkaConsumerFactory<>(props); + factory.setBeanName(beanName); + if (this.clientFactoryCustomizer != null) { + this.clientFactoryCustomizer.configure(factory); + } + return factory; + } + + private boolean isAutoCommitOnError( + ExtendedConsumerProperties properties) { + return properties.getExtension().getAutoCommitOnError() != null + ? properties.getExtension().getAutoCommitOnError() + : false; + } + + private TopicPartitionOffset[] getTopicPartitionOffsets( + Collection listenedPartitions, + ExtendedConsumerProperties extendedConsumerProperties, + ConsumerFactory consumerFactory) { + + final TopicPartitionOffset[] TopicPartitionOffsets = + new TopicPartitionOffset[listenedPartitions.size()]; + int i = 0; + SeekPosition seekPosition = null; + Object resetTo = checkReset(extendedConsumerProperties.getExtension().isResetOffsets(), + consumerFactory.getConfigurationProperties().get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)); + if (resetTo != null) { + seekPosition = "earliest".equals(resetTo) ? SeekPosition.BEGINNING : SeekPosition.END; + } + for (PartitionInfo partition : listenedPartitions) { + + TopicPartitionOffsets[i++] = new TopicPartitionOffset( + partition.topic(), partition.partition(), seekPosition); + } + return TopicPartitionOffsets; + } + + private String toDisplayString(String original, int maxCharacters) { + if (original.length() <= maxCharacters) { + return original; + } + return original.substring(0, maxCharacters) + "..."; + } + + private String getStackTraceAsString(Throwable cause) { + StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter, true); + cause.printStackTrace(printWriter); + return stringWriter.getBuffer().toString(); + } + + public void setConsumerConfigCustomizer(ConsumerConfigCustomizer consumerConfigCustomizer) { + this.consumerConfigCustomizer = consumerConfigCustomizer; + } + + public void setProducerConfigCustomizer(ProducerConfigCustomizer producerConfigCustomizer) { + this.producerConfigCustomizer = producerConfigCustomizer; + } + + List> getKafkaMessageListenerContainers() { + return Collections.unmodifiableList(kafkaMessageListenerContainers); + } + + private final class ProducerConfigurationMessageHandler + extends KafkaProducerMessageHandler { + + private boolean running = true; + + private final ProducerFactory producerFactory; + + ProducerConfigurationMessageHandler(KafkaTemplate kafkaTemplate, + String topic, + ExtendedProducerProperties producerProperties, + ProducerFactory producerFactory) { + + super(kafkaTemplate); + if (producerProperties.getExtension().isUseTopicHeader()) { + setTopicExpression(PARSER.parseExpression("headers['" + KafkaHeaders.TOPIC + "'] ?: '" + topic + "'")); + } + else { + setTopicExpression(new LiteralExpression(topic)); + } + Expression messageKeyExpression = producerProperties.getExtension().getMessageKeyExpression(); + if (expressionInterceptorNeeded(producerProperties)) { + messageKeyExpression = PARSER.parseExpression("headers['" + + KafkaExpressionEvaluatingInterceptor.MESSAGE_KEY_HEADER + + "']"); + } + setMessageKeyExpression(messageKeyExpression); + setBeanFactory(KafkaMessageChannelBinder.this.getBeanFactory()); + if (producerProperties.isPartitioned()) { + setPartitionIdExpression(PARSER.parseExpression( + "headers['" + BinderHeaders.PARTITION_HEADER + "']")); + } + if (producerProperties.getExtension().isSync()) { + setSync(true); + } + if (producerProperties.getExtension().getSendTimeoutExpression() != null) { + setSendTimeoutExpression(producerProperties.getExtension().getSendTimeoutExpression()); + } + this.producerFactory = producerFactory; + } + + @Override + public void start() { + try { + super.onInit(); + } + catch (Exception ex) { + this.logger.error(ex, "Initialization errors: "); + throw new RuntimeException(ex); + } + } + + @Override + public void stop() { + if (this.producerFactory instanceof DisposableBean) { + try { + ((DisposableBean) producerFactory).destroy(); + } + catch (Exception ex) { + this.logger.error(ex, "Error destroying the producer factory bean: "); + throw new RuntimeException(ex); + } + } + this.running = false; + } + + @Override + public boolean isRunning() { + return this.running; + } + + } + + /** + * Inner class to capture topic details. + */ + static class TopicInformation { + + private final String consumerGroup; + + private final Collection partitionInfos; + + private final boolean isTopicPattern; + + TopicInformation(String consumerGroup, Collection partitionInfos, + boolean isTopicPattern) { + this.consumerGroup = consumerGroup; + this.partitionInfos = partitionInfos; + this.isTopicPattern = isTopicPattern; + } + + String getConsumerGroup() { + return this.consumerGroup; + } + + boolean isConsumerTopic() { + return this.consumerGroup != null; + } + + boolean isTopicPattern() { + return this.isTopicPattern; + } + + Collection getPartitionInfos() { + return this.partitionInfos; + } + + } + + /** + * Helper class to send to DLQ. + * + * @param generic type for key + * @param generic type for value + */ + private final class DlqSender { + + private final KafkaTemplate kafkaTemplate; + + private final long sendTimeout; + + DlqSender(KafkaTemplate kafkaTemplate, long timeout) { + this.kafkaTemplate = kafkaTemplate; + this.sendTimeout = timeout; + } + + @SuppressWarnings("unchecked") + void sendToDlq(ConsumerRecord consumerRecord, Headers headers, + String dlqName, String group, Throwable throwable, DlqPartitionFunction partitionFunction, + MessageHeaders messageHeaders, ContainerProperties.AckMode ackMode) { + K key = (K) consumerRecord.key(); + V value = (V) consumerRecord.value(); + ProducerRecord producerRecord = new ProducerRecord<>(dlqName, + partitionFunction.apply(group, consumerRecord, throwable), + key, value, headers); + + StringBuilder sb = new StringBuilder().append(" a message with key='") + .append(keyOrValue(key)) + .append("'").append(" and payload='") + .append(keyOrValue(value)) + .append("'").append(" received from ") + .append(consumerRecord.partition()); + ListenableFuture> sentDlq = null; + try { + sentDlq = this.kafkaTemplate.send(producerRecord); + sentDlq.addCallback(new ListenableFutureCallback>() { + + @Override + public void onFailure(Throwable ex) { + KafkaMessageChannelBinder.this.logger + .error("Error sending to DLQ " + sb.toString(), ex); + } + + @Override + public void onSuccess(SendResult result) { + if (KafkaMessageChannelBinder.this.logger.isDebugEnabled()) { + KafkaMessageChannelBinder.this.logger + .debug("Sent to DLQ " + sb.toString() + ": " + result.getRecordMetadata()); + } + } + }); + try { + sentDlq.get(this.sendTimeout, TimeUnit.MILLISECONDS); + } + catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw ex; + } + } + catch (Exception ex) { + KafkaMessageChannelBinder.this.logger + .error("Error sending to DLQ " + sb.toString(), ex); + } + finally { + if (ackMode == ContainerProperties.AckMode.MANUAL + || ackMode == ContainerProperties.AckMode.MANUAL_IMMEDIATE) { + messageHeaders.get(KafkaHeaders.ACKNOWLEDGMENT, Acknowledgment.class).acknowledge(); + } + } + } + + private String keyOrValue(Object keyOrValue) { + if (keyOrValue instanceof byte[]) { + return "byte[" + ((byte[]) keyOrValue).length + "]"; + } + else { + return toDisplayString(ObjectUtils.nullSafeToString(keyOrValue), 50); + } + } + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaNullConverter.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaNullConverter.java new file mode 100644 index 000000000..713dbf338 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaNullConverter.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.Collections; + +import org.springframework.kafka.support.KafkaNull; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.converter.AbstractMessageConverter; +import org.springframework.messaging.converter.MessageConverter; +import org.springframework.util.MimeTypeUtils; + +/** + * A {@link MessageConverter} that supports {@link KafkaNull} payloads. + * + * @author Gary Russell + * @author Aldo Sinanaj + * @since 2.2 + */ +public class KafkaNullConverter extends AbstractMessageConverter { + + public KafkaNullConverter() { + super(Collections.singletonList(MimeTypeUtils.ALL)); + } + + @Override + protected boolean supportsMimeType(MessageHeaders headers) { + return true; + } + + @Override + protected boolean supports(Class aClass) { + return KafkaNull.class.equals(aClass); + } + + @Override + protected boolean canConvertFrom(Message message, Class targetClass) { + return message.getPayload() instanceof KafkaNull; + } + + @Override + protected Object convertFromInternal(Message message, Class targetClass, + Object conversionHint) { + return message.getPayload(); + } + + @Override + protected Object convertToInternal(Object payload, MessageHeaders headers, + Object conversionHint) { + return payload; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/ListenerContainerWithDlqAndRetryCustomizer.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/ListenerContainerWithDlqAndRetryCustomizer.java new file mode 100644 index 000000000..567ac6441 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/ListenerContainerWithDlqAndRetryCustomizer.java @@ -0,0 +1,71 @@ +/* + * Copyright 2021-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.function.BiFunction; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.TopicPartition; + +import org.springframework.cloud.stream.config.ListenerContainerCustomizer; +import org.springframework.kafka.listener.AbstractMessageListenerContainer; +import org.springframework.lang.Nullable; +import org.springframework.util.backoff.BackOff; + +/** + * An extension of {@link ListenerContainerCustomizer} that provides access to dead letter + * metadata. + * + * @author Gary Russell + * @since 3.2 + * + */ +public interface ListenerContainerWithDlqAndRetryCustomizer + extends ListenerContainerCustomizer> { + + @Override + default void configure(AbstractMessageListenerContainer container, String destinationName, String group) { + } + + /** + * Configure the container. + * @param container the container. + * @param destinationName the destination name. + * @param group the group. + * @param dlqDestinationResolver a destination resolver for the dead letter topic (if + * enableDlq). + * @param backOff the backOff using retry properties (if configured). + * @see #retryAndDlqInBinding(String, String) + */ + void configure(AbstractMessageListenerContainer container, String destinationName, String group, + @Nullable BiFunction, Exception, TopicPartition> dlqDestinationResolver, + @Nullable BackOff backOff); + + /** + * Return false to move retries and DLQ from the binding to a customized error handler + * using the retry metadata and/or a {@code DeadLetterPublishingRecoverer} when + * configured via + * {@link #configure(AbstractMessageListenerContainer, String, String, BiFunction, BackOff)}. + * @param destinationName the destination name. + * @param group the group. + * @return true to disable retrie in the binding + */ + default boolean retryAndDlqInBinding(String destinationName, String group) { + return true; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ClientFactoryCustomizer.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ClientFactoryCustomizer.java new file mode 100644 index 000000000..d2583dc5a --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ClientFactoryCustomizer.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.config; + +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.ProducerFactory; + +/** + * Called by the binder to customize the factories. + * + * @author Gary Russell + * @since 3.0.6 + * + */ +public interface ClientFactoryCustomizer { + + default void configure(ProducerFactory pf) { + } + + default void configure(ConsumerFactory cf) { + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ConsumerConfigCustomizer.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ConsumerConfigCustomizer.java new file mode 100644 index 000000000..adc184803 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ConsumerConfigCustomizer.java @@ -0,0 +1,32 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.config; + +import java.util.Map; + +/** + * This customizer is called by the binder to customize consumer configuration in + * Kafka Consumer factory. + * + * @author Soby Chacko + * @since 3.0.9 + */ +@FunctionalInterface +public interface ConsumerConfigCustomizer { + + void configure(Map consumerProperties, String bindingName, String destination); +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ExtendedBindingHandlerMappingsProviderConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ExtendedBindingHandlerMappingsProviderConfiguration.java new file mode 100644 index 000000000..c05c715e3 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ExtendedBindingHandlerMappingsProviderConfiguration.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.config; + +import java.util.HashMap; +import java.util.Map; + +import org.springframework.boot.context.properties.source.ConfigurationPropertyName; +import org.springframework.cloud.stream.config.BindingHandlerAdvise.MappingsProvider; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Configuration for extended binding metadata. + * + * @author Oleg Zhurakousky + */ + +@Configuration +public class ExtendedBindingHandlerMappingsProviderConfiguration { + + @Bean + public MappingsProvider kafkaExtendedPropertiesDefaultMappingsProvider() { + return () -> { + Map mappings = new HashMap<>(); + mappings.put( + ConfigurationPropertyName.of("spring.cloud.stream.kafka.bindings"), + ConfigurationPropertyName.of("spring.cloud.stream.kafka.default")); + return mappings; + }; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java new file mode 100644 index 000000000..b8d03225c --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java @@ -0,0 +1,288 @@ +/* + * Copyright 2015-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.config; + +import java.io.IOException; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.binder.MeterBinder; + +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.stream.annotation.StreamMessageConverter; +import org.springframework.cloud.stream.binder.Binder; +import org.springframework.cloud.stream.binder.kafka.KafkaBinderMetrics; +import org.springframework.cloud.stream.binder.kafka.KafkaBindingRebalanceListener; +import org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder; +import org.springframework.cloud.stream.binder.kafka.KafkaNullConverter; +import org.springframework.cloud.stream.binder.kafka.properties.JaasLoginModuleConfiguration; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaExtendedBindingProperties; +import org.springframework.cloud.stream.binder.kafka.provisioning.AdminClientConfigCustomizer; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.cloud.stream.binder.kafka.utils.DlqDestinationResolver; +import org.springframework.cloud.stream.binder.kafka.utils.DlqPartitionFunction; +import org.springframework.cloud.stream.config.ConsumerEndpointCustomizer; +import org.springframework.cloud.stream.config.ListenerContainerCustomizer; +import org.springframework.cloud.stream.config.MessageSourceCustomizer; +import org.springframework.cloud.stream.config.ProducerMessageHandlerCustomizer; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.inbound.KafkaMessageSource; +import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.MicrometerConsumerListener; +import org.springframework.kafka.core.MicrometerProducerListener; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.AbstractMessageListenerContainer; +import org.springframework.kafka.security.jaas.KafkaJaasLoginModuleInitializer; +import org.springframework.kafka.support.LoggingProducerListener; +import org.springframework.kafka.support.ProducerListener; +import org.springframework.lang.Nullable; +import org.springframework.messaging.converter.MessageConverter; + +/** + * Kafka binder configuration class. + * + * @author David Turanski + * @author Marius Bogoevici + * @author Soby Chacko + * @author Mark Fisher + * @author Ilayaperumal Gopinathan + * @author Henryk Konsek + * @author Gary Russell + * @author Oleg Zhurakousky + * @author Artem Bilan + * @author Aldo Sinanaj + */ +@Configuration(proxyBeanMethods = false) +@ConditionalOnMissingBean(Binder.class) +@Import({ KafkaAutoConfiguration.class, KafkaBinderHealthIndicatorConfiguration.class }) +@EnableConfigurationProperties({ KafkaExtendedBindingProperties.class }) +public class KafkaBinderConfiguration { + + @Bean + KafkaBinderConfigurationProperties configurationProperties( + KafkaProperties kafkaProperties) { + return new KafkaBinderConfigurationProperties(kafkaProperties); + } + + @Bean + KafkaTopicProvisioner provisioningProvider( + KafkaBinderConfigurationProperties configurationProperties, + ObjectProvider adminClientConfigCustomizer, KafkaProperties kafkaProperties) { + return new KafkaTopicProvisioner(configurationProperties, + kafkaProperties, adminClientConfigCustomizer.getIfUnique()); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + @Bean + KafkaMessageChannelBinder kafkaMessageChannelBinder( + KafkaBinderConfigurationProperties configurationProperties, + KafkaTopicProvisioner provisioningProvider, + @Nullable ListenerContainerCustomizer> listenerContainerCustomizer, + @Nullable MessageSourceCustomizer> sourceCustomizer, + @Nullable ProducerMessageHandlerCustomizer> messageHandlerCustomizer, + @Nullable ConsumerEndpointCustomizer> consumerCustomizer, + ObjectProvider rebalanceListener, + ObjectProvider dlqPartitionFunction, + ObjectProvider dlqDestinationResolver, + ObjectProvider clientFactoryCustomizer, + ObjectProvider consumerConfigCustomizer, + ObjectProvider producerConfigCustomizer, + ProducerListener producerListener, KafkaExtendedBindingProperties kafkaExtendedBindingProperties + ) { + + KafkaMessageChannelBinder kafkaMessageChannelBinder = new KafkaMessageChannelBinder( + configurationProperties, provisioningProvider, + listenerContainerCustomizer, sourceCustomizer, rebalanceListener.getIfUnique(), + dlqPartitionFunction.getIfUnique(), dlqDestinationResolver.getIfUnique()); + kafkaMessageChannelBinder.setProducerListener(producerListener); + kafkaMessageChannelBinder + .setExtendedBindingProperties(kafkaExtendedBindingProperties); + kafkaMessageChannelBinder.setProducerMessageHandlerCustomizer(messageHandlerCustomizer); + kafkaMessageChannelBinder.setConsumerEndpointCustomizer(consumerCustomizer); + kafkaMessageChannelBinder.setClientFactoryCustomizer(clientFactoryCustomizer.getIfUnique()); + kafkaMessageChannelBinder.setConsumerConfigCustomizer(consumerConfigCustomizer.getIfUnique()); + kafkaMessageChannelBinder.setProducerConfigCustomizer(producerConfigCustomizer.getIfUnique()); + return kafkaMessageChannelBinder; + } + + @SuppressWarnings("rawtypes") + @Bean + @ConditionalOnMissingBean(ProducerListener.class) + ProducerListener producerListener() { + return new LoggingProducerListener(); + } + + @Bean + @StreamMessageConverter + @ConditionalOnMissingBean(KafkaNullConverter.class) + MessageConverter kafkaNullConverter() { + return new KafkaNullConverter(); + } + + @Bean + @ConditionalOnMissingBean(KafkaJaasLoginModuleInitializer.class) + public KafkaJaasLoginModuleInitializer jaasInitializer( + KafkaBinderConfigurationProperties configurationProperties) + throws IOException { + KafkaJaasLoginModuleInitializer kafkaJaasLoginModuleInitializer = new KafkaJaasLoginModuleInitializer(); + JaasLoginModuleConfiguration jaas = configurationProperties.getJaas(); + if (jaas != null) { + kafkaJaasLoginModuleInitializer.setLoginModule(jaas.getLoginModule()); + + KafkaJaasLoginModuleInitializer.ControlFlag controlFlag = jaas + .getControlFlag(); + + if (controlFlag != null) { + kafkaJaasLoginModuleInitializer.setControlFlag(controlFlag); + } + kafkaJaasLoginModuleInitializer.setOptions(jaas.getOptions()); + } + return kafkaJaasLoginModuleInitializer; + } + + @Configuration + @ConditionalOnMissingBean(value = KafkaBinderMetrics.class, name = "outerContext") + @ConditionalOnClass(name = "io.micrometer.core.instrument.MeterRegistry") + protected class KafkaBinderMetricsConfiguration { + + @Bean + @ConditionalOnBean(MeterRegistry.class) + @ConditionalOnMissingBean(KafkaBinderMetrics.class) + public MeterBinder kafkaBinderMetrics( + KafkaMessageChannelBinder kafkaMessageChannelBinder, + KafkaBinderConfigurationProperties configurationProperties, + MeterRegistry meterRegistry) { + + return new KafkaBinderMetrics(kafkaMessageChannelBinder, + configurationProperties, null, meterRegistry); + } + + @ConditionalOnClass(name = "org.springframework.kafka.core.MicrometerConsumerListener") + @ConditionalOnBean(MeterRegistry.class) + protected class KafkaMicrometer { + + @Bean + @ConditionalOnMissingBean(name = "binderClientFactoryCustomizer") + public ClientFactoryCustomizer binderClientFactoryCustomizer(MeterRegistry meterRegistry) { + + return new ClientFactoryCustomizer() { + + @Override + public void configure(ProducerFactory pf) { + if (pf instanceof DefaultKafkaProducerFactory) { + ((DefaultKafkaProducerFactory) pf) + .addListener(new MicrometerProducerListener<>(meterRegistry)); + } + } + + @Override + public void configure(ConsumerFactory cf) { + if (cf instanceof DefaultKafkaConsumerFactory) { + ((DefaultKafkaConsumerFactory) cf) + .addListener(new MicrometerConsumerListener<>(meterRegistry)); + } + } + + }; + + } + + } + + } + + @Configuration + @ConditionalOnBean(name = "outerContext") + @ConditionalOnMissingBean(KafkaBinderMetrics.class) + @ConditionalOnClass(name = "io.micrometer.core.instrument.MeterRegistry") + protected class KafkaBinderMetricsConfigurationWithMultiBinder { + + @Bean + public MeterBinder kafkaBinderMetrics( + KafkaMessageChannelBinder kafkaMessageChannelBinder, + KafkaBinderConfigurationProperties configurationProperties, + ConfigurableApplicationContext context) { + + MeterRegistry meterRegistry = context.getBean("outerContext", ApplicationContext.class) + .getBean(MeterRegistry.class); + return new KafkaBinderMetrics(kafkaMessageChannelBinder, + configurationProperties, null, meterRegistry); + } + + @ConditionalOnClass(name = "org.springframework.kafka.core.MicrometerConsumerListener") + protected class KafkaMicrometer { + + @Bean + @ConditionalOnMissingBean(name = "binderClientFactoryCustomizer") + public ClientFactoryCustomizer binderClientFactoryCustomizer(ConfigurableApplicationContext context) { + + + return new ClientFactoryCustomizer() { + + MeterRegistry meterRegistry = context.getBean("outerContext", ApplicationContext.class) + .getBean(MeterRegistry.class); + + @Override + public void configure(ProducerFactory pf) { + if (pf instanceof DefaultKafkaProducerFactory) { + ((DefaultKafkaProducerFactory) pf) + .addListener(new MicrometerProducerListener<>(this.meterRegistry)); + } + } + + @Override + public void configure(ConsumerFactory cf) { + if (cf instanceof DefaultKafkaConsumerFactory) { + ((DefaultKafkaConsumerFactory) cf) + .addListener(new MicrometerConsumerListener<>(this.meterRegistry)); + } + } + + }; + + } + + } + + } + + /** + * Properties configuration for Jaas. + */ + @SuppressWarnings("unused") + public static class JaasConfigurationProperties { + + private JaasLoginModuleConfiguration kafka; + + private JaasLoginModuleConfiguration zookeeper; + + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderHealthIndicatorConfiguration.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderHealthIndicatorConfiguration.java new file mode 100644 index 000000000..67d53a182 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderHealthIndicatorConfiguration.java @@ -0,0 +1,78 @@ +/* + * Copyright 2018-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.config; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; + +import org.springframework.boot.actuate.autoconfigure.health.ConditionalOnEnabledHealthIndicator; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealth; +import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealthIndicator; +import org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.util.ObjectUtils; + +/** + * Configuration class for Kafka binder health indicator beans. + * + * @author Oleg Zhurakousky + * @author Chukwubuikem Ume-Ugwa + * @author Soby Chacko + */ + +@Configuration(proxyBeanMethods = false) +@ConditionalOnClass(name = "org.springframework.boot.actuate.health.HealthIndicator") +@ConditionalOnEnabledHealthIndicator("binders") +@ConditionalOnMissingBean(KafkaBinderHealth.class) +public class KafkaBinderHealthIndicatorConfiguration { + + @Bean + public KafkaBinderHealthIndicator kafkaBinderHealthIndicator( + KafkaMessageChannelBinder kafkaMessageChannelBinder, + KafkaBinderConfigurationProperties configurationProperties) { + Map props = new HashMap<>(); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + ByteArrayDeserializer.class); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + ByteArrayDeserializer.class); + Map mergedConfig = configurationProperties + .mergedConsumerConfiguration(); + if (!ObjectUtils.isEmpty(mergedConfig)) { + props.putAll(mergedConfig); + } + if (!props.containsKey(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)) { + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + configurationProperties.getKafkaConnectionString()); + } + ConsumerFactory consumerFactory = new DefaultKafkaConsumerFactory<>(props); + KafkaBinderHealthIndicator indicator = new KafkaBinderHealthIndicator( + kafkaMessageChannelBinder, consumerFactory); + indicator.setTimeout(configurationProperties.getHealthTimeout()); + indicator.setConsiderDownWhenAnyPartitionHasNoLeader(configurationProperties.isConsiderDownWhenAnyPartitionHasNoLeader()); + return indicator; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ProducerConfigCustomizer.java b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ProducerConfigCustomizer.java new file mode 100644 index 000000000..9dd5cd9d6 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ProducerConfigCustomizer.java @@ -0,0 +1,32 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.config; + +import java.util.Map; + +/** + * This customizer is called by the binder to customize producer configuration in + * Kafka Producer factory. + * + * @author Soby Chacko + * @since 3.0.9 + */ +@FunctionalInterface +public interface ProducerConfigCustomizer { + + void configure(Map producerProperties, String bindingName, String destination); +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/resources/META-INF/spring.binders b/k-binder/spring-cloud-stream-binder-kafka/src/main/resources/META-INF/spring.binders new file mode 100644 index 000000000..063a7400f --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/resources/META-INF/spring.binders @@ -0,0 +1,2 @@ +kafka:\ +org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfiguration diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/main/resources/META-INF/spring.factories b/k-binder/spring-cloud-stream-binder-kafka/src/main/resources/META-INF/spring.factories new file mode 100644 index 000000000..1d2e61ded --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/main/resources/META-INF/spring.factories @@ -0,0 +1,4 @@ +org.springframework.boot.env.EnvironmentPostProcessor:\ +org.springframework.cloud.stream.binder.kafka.KafkaBinderEnvironmentPostProcessor +org.springframework.boot.autoconfigure.EnableAutoConfiguration:\ +org.springframework.cloud.stream.binder.kafka.config.ExtendedBindingHandlerMappingsProviderConfiguration diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/AbstractKafkaTestBinder.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/AbstractKafkaTestBinder.java new file mode 100644 index 000000000..2035a5729 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/AbstractKafkaTestBinder.java @@ -0,0 +1,50 @@ +/* + * Copyright 2014-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import org.springframework.cloud.stream.binder.AbstractPollableConsumerTestBinder; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; +import org.springframework.context.ApplicationContext; + +/** + * @author Soby Chacko + * @author Gary Russell + */ +public abstract class AbstractKafkaTestBinder extends + // @checkstyle:off + AbstractPollableConsumerTestBinder, ExtendedProducerProperties> { + + // @checkstyle:on + private ApplicationContext applicationContext; + + @Override + public void cleanup() { + // do nothing - the rule will take care of that + } + + protected final void setApplicationContext(ApplicationContext context) { + this.applicationContext = context; + } + + public ApplicationContext getApplicationContext() { + return this.applicationContext; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/AdminConfigTests.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/AdminConfigTests.java new file mode 100644 index 000000000..8bbad166f --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/AdminConfigTests.java @@ -0,0 +1,89 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.Arrays; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfiguration; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaTopicProperties; +import org.springframework.cloud.stream.config.BindingServiceConfiguration; +import org.springframework.integration.config.EnableIntegration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @author Aldo Sinanaj + * @since 2.0 + * + */ +@RunWith(SpringRunner.class) +@SpringBootTest(classes = { KafkaBinderConfiguration.class, + BindingServiceConfiguration.class }) +@TestPropertySource(properties = { + "spring.cloud.stream.kafka.bindings.input.consumer.topic.replication-factor=2", + "spring.cloud.stream.kafka.bindings.input.consumer.topic.replicas-assignments.0=0,1", + "spring.cloud.stream.kafka.bindings.input.consumer.topic.properties.message.format.version=0.9.0.0", + "spring.cloud.stream.kafka.bindings.secondInput.consumer.topic.replication-factor=3", + "spring.cloud.stream.kafka.bindings.secondInput.consumer.topic.replicas-assignments.0=0,1", + "spring.cloud.stream.kafka.bindings.secondInput.consumer.topic.properties.message.format.version=0.9.1.0", + "spring.cloud.stream.kafka.bindings.output.producer.topic.replication-factor=2", + "spring.cloud.stream.kafka.bindings.output.producer.topic.replicas-assignments.0=0,1", + "spring.cloud.stream.kafka.bindings.output.producer.topic.properties.message.format.version=0.9.0.0", + "spring.main.allow-bean-definition-overriding=true" }) +@EnableIntegration +public class AdminConfigTests { + + @Autowired + private KafkaMessageChannelBinder binder; + + @Test + public void testConsumerTopicProperties() { + final KafkaConsumerProperties consumerProperties = this.binder + .getExtendedConsumerProperties("secondInput"); + final KafkaTopicProperties kafkaTopicProperties = consumerProperties.getTopic(); + + assertThat(kafkaTopicProperties.getReplicationFactor()).isEqualTo((short) 3); + assertThat(kafkaTopicProperties.getReplicasAssignments().get(0)) + .isEqualTo(Arrays.asList(0, 1)); + assertThat(kafkaTopicProperties.getProperties().get("message.format.version")) + .isEqualTo("0.9.1.0"); + } + + @Test + public void testProducerTopicProperties() { + final KafkaProducerProperties producerProperties = this.binder + .getExtendedProducerProperties("output"); + final KafkaTopicProperties kafkaTopicProperties = producerProperties.getTopic(); + + assertThat(kafkaTopicProperties.getReplicationFactor()).isEqualTo((short) 2); + assertThat(kafkaTopicProperties.getReplicasAssignments().get(0)) + .isEqualTo(Arrays.asList(0, 1)); + assertThat(kafkaTopicProperties.getProperties().get("message.format.version")) + .isEqualTo("0.9.0.0"); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/AutoCreateTopicDisabledTests.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/AutoCreateTopicDisabledTests.java new file mode 100644 index 000000000..672a020a0 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/AutoCreateTopicDisabledTests.java @@ -0,0 +1,121 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.Collections; + +import kafka.server.KafkaConfig; +import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.cloud.stream.binder.BinderException; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.retry.policy.SimpleRetryPolicy; +import org.springframework.retry.support.RetryTemplate; + +import static org.hamcrest.CoreMatchers.isA; + +/** + * @author Soby Chacko + */ +public class AutoCreateTopicDisabledTests { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @ClassRule + public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, 1) + .brokerProperty(KafkaConfig.AutoCreateTopicsEnableProp(), "false"); + + @Test + public void testAutoCreateTopicDisabledFailsOnConsumerIfTopicNonExistentOnBroker() + throws Throwable { + + KafkaProperties kafkaProperties = new TestKafkaProperties(); + kafkaProperties.setBootstrapServers(Collections + .singletonList(embeddedKafka.getEmbeddedKafka().getBrokersAsString())); + KafkaBinderConfigurationProperties configurationProperties = new KafkaBinderConfigurationProperties( + kafkaProperties); + // disable auto create topic on the binder. + configurationProperties.setAutoCreateTopics(false); + + KafkaTopicProvisioner provisioningProvider = new KafkaTopicProvisioner( + configurationProperties, kafkaProperties, null); + provisioningProvider.setMetadataRetryOperations(new RetryTemplate()); + + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder( + configurationProperties, provisioningProvider); + + final String testTopicName = "nonExistent" + System.currentTimeMillis(); + + ExtendedConsumerProperties properties = new ExtendedConsumerProperties<>( + new KafkaConsumerProperties()); + + expectedException.expect(BinderException.class); + expectedException.expectCause(isA(UnknownTopicOrPartitionException.class)); + binder.createConsumerEndpoint(() -> testTopicName, "group", properties); + } + + @Test + public void testAutoCreateTopicDisabledFailsOnProducerIfTopicNonExistentOnBroker() + throws Throwable { + + KafkaProperties kafkaProperties = new TestKafkaProperties(); + kafkaProperties.setBootstrapServers(Collections + .singletonList(embeddedKafka.getEmbeddedKafka().getBrokersAsString())); + + KafkaBinderConfigurationProperties configurationProperties = new KafkaBinderConfigurationProperties( + kafkaProperties); + // disable auto create topic on the binder. + configurationProperties.setAutoCreateTopics(false); + // reduce the wait time on the producer blocking operations. + configurationProperties.getConfiguration().put("max.block.ms", "3000"); + + KafkaTopicProvisioner provisioningProvider = new KafkaTopicProvisioner( + configurationProperties, kafkaProperties, null); + SimpleRetryPolicy simpleRetryPolicy = new SimpleRetryPolicy(1); + final RetryTemplate metadataRetryOperations = new RetryTemplate(); + metadataRetryOperations.setRetryPolicy(simpleRetryPolicy); + provisioningProvider.setMetadataRetryOperations(metadataRetryOperations); + + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder( + configurationProperties, provisioningProvider); + + final String testTopicName = "nonExistent" + System.currentTimeMillis(); + + ExtendedProducerProperties properties = new ExtendedProducerProperties<>( + new KafkaProducerProperties()); + + expectedException.expect(BinderException.class); + expectedException.expectCause(isA(UnknownTopicOrPartitionException.class)); + + binder.bindProducer(testTopicName, new DirectChannel(), properties); + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderAutoConfigurationPropertiesTest.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderAutoConfigurationPropertiesTest.java new file mode 100644 index 000000000..fffacd886 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderAutoConfigurationPropertiesTest.java @@ -0,0 +1,137 @@ +/* + * Copyright 2016-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.kafka.common.serialization.LongDeserializer; +import org.apache.kafka.common.serialization.LongSerializer; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfiguration; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.util.ReflectionUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Ilayaperumal Gopinathan + */ +@RunWith(SpringJUnit4ClassRunner.class) +@SpringBootTest(classes = { KafkaBinderConfiguration.class }) +@TestPropertySource(locations = "classpath:binder-config-autoconfig.properties") +public class KafkaBinderAutoConfigurationPropertiesTest { + + @Autowired + private KafkaMessageChannelBinder kafkaMessageChannelBinder; + + @Autowired + private KafkaBinderHealthIndicator kafkaBinderHealthIndicator; + + @Test + @SuppressWarnings("unchecked") + public void testKafkaBinderConfigurationWithKafkaProperties() throws Exception { + assertThat(this.kafkaMessageChannelBinder).isNotNull(); + ExtendedProducerProperties producerProperties = new ExtendedProducerProperties<>( + new KafkaProducerProperties()); + Method getProducerFactoryMethod = KafkaMessageChannelBinder.class + .getDeclaredMethod("getProducerFactory", String.class, + ExtendedProducerProperties.class, String.class, String.class); + getProducerFactoryMethod.setAccessible(true); + DefaultKafkaProducerFactory producerFactory = (DefaultKafkaProducerFactory) getProducerFactoryMethod + .invoke(this.kafkaMessageChannelBinder, "foo", producerProperties, "foo.producer", "foo"); + Field producerFactoryConfigField = ReflectionUtils + .findField(DefaultKafkaProducerFactory.class, "configs", Map.class); + ReflectionUtils.makeAccessible(producerFactoryConfigField); + Map producerConfigs = (Map) ReflectionUtils + .getField(producerFactoryConfigField, producerFactory); + assertThat(producerConfigs.get("batch.size").equals(10)).isTrue(); + assertThat(producerConfigs.get("key.serializer")).isEqualTo(LongSerializer.class); + assertThat(producerConfigs.get("key.deserializer")).isNull(); + assertThat(producerConfigs.get("value.serializer")) + .isEqualTo(LongSerializer.class); + assertThat(producerConfigs.get("value.deserializer")).isNull(); + assertThat(producerConfigs.get("compression.type")).isEqualTo("snappy"); + List bootstrapServers = new ArrayList<>(); + bootstrapServers.add("10.98.09.199:9092"); + bootstrapServers.add("10.98.09.196:9092"); + assertThat((((List) producerConfigs.get("bootstrap.servers")) + .containsAll(bootstrapServers))).isTrue(); + Method createKafkaConsumerFactoryMethod = KafkaMessageChannelBinder.class + .getDeclaredMethod("createKafkaConsumerFactory", boolean.class, + String.class, ExtendedConsumerProperties.class, String.class, String.class); + createKafkaConsumerFactoryMethod.setAccessible(true); + ExtendedConsumerProperties consumerProperties = new ExtendedConsumerProperties<>( + new KafkaConsumerProperties()); + DefaultKafkaConsumerFactory consumerFactory = (DefaultKafkaConsumerFactory) createKafkaConsumerFactoryMethod + .invoke(this.kafkaMessageChannelBinder, true, "test", consumerProperties, "test.consumer", "test"); + Field consumerFactoryConfigField = ReflectionUtils + .findField(DefaultKafkaConsumerFactory.class, "configs", Map.class); + ReflectionUtils.makeAccessible(consumerFactoryConfigField); + Map consumerConfigs = (Map) ReflectionUtils + .getField(consumerFactoryConfigField, consumerFactory); + assertThat(consumerConfigs.get("key.deserializer")) + .isEqualTo(LongDeserializer.class); + assertThat(consumerConfigs.get("key.serializer")).isNull(); + assertThat(consumerConfigs.get("value.deserializer")) + .isEqualTo(LongDeserializer.class); + assertThat(consumerConfigs.get("value.serialized")).isNull(); + assertThat(consumerConfigs.get("group.id")).isEqualTo("test"); + assertThat(consumerConfigs.get("auto.offset.reset")).isEqualTo("earliest"); + assertThat((((List) consumerConfigs.get("bootstrap.servers")) + .containsAll(bootstrapServers))).isTrue(); + } + + @Test + @SuppressWarnings("unchecked") + public void testKafkaHealthIndicatorProperties() { + assertThat(this.kafkaBinderHealthIndicator).isNotNull(); + Field consumerFactoryField = ReflectionUtils.findField( + KafkaBinderHealthIndicator.class, "consumerFactory", + ConsumerFactory.class); + ReflectionUtils.makeAccessible(consumerFactoryField); + DefaultKafkaConsumerFactory consumerFactory = (DefaultKafkaConsumerFactory) ReflectionUtils + .getField(consumerFactoryField, this.kafkaBinderHealthIndicator); + Field configField = ReflectionUtils.findField(DefaultKafkaConsumerFactory.class, + "configs", Map.class); + ReflectionUtils.makeAccessible(configField); + Map configs = (Map) ReflectionUtils + .getField(configField, consumerFactory); + assertThat(configs.containsKey("bootstrap.servers")).isTrue(); + List bootstrapServers = new ArrayList<>(); + bootstrapServers.add("10.98.09.199:9092"); + bootstrapServers.add("10.98.09.196:9092"); + assertThat(((List) configs.get("bootstrap.servers")) + .containsAll(bootstrapServers)).isTrue(); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderConfigurationPropertiesTest.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderConfigurationPropertiesTest.java new file mode 100644 index 000000000..f761bd503 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderConfigurationPropertiesTest.java @@ -0,0 +1,122 @@ +/* + * Copyright 2016-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfiguration; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.util.ReflectionUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Ilayaperumal Gopinathan + */ +@RunWith(SpringJUnit4ClassRunner.class) +@SpringBootTest(classes = { KafkaBinderConfiguration.class, KafkaAutoConfiguration.class, + KafkaBinderConfigurationPropertiesTest.class }) +@TestPropertySource(locations = "classpath:binder-config.properties") +public class KafkaBinderConfigurationPropertiesTest { + + @Autowired + private KafkaMessageChannelBinder kafkaMessageChannelBinder; + + @Test + @SuppressWarnings("unchecked") + public void testKafkaBinderConfigurationProperties() throws Exception { + assertThat(this.kafkaMessageChannelBinder).isNotNull(); + KafkaProducerProperties kafkaProducerProperties = new KafkaProducerProperties(); + kafkaProducerProperties.setBufferSize(12345); + kafkaProducerProperties.setBatchTimeout(100); + kafkaProducerProperties.setCloseTimeout(10); + kafkaProducerProperties + .setCompressionType(KafkaProducerProperties.CompressionType.gzip); + ExtendedProducerProperties producerProperties = new ExtendedProducerProperties<>( + kafkaProducerProperties); + Method getProducerFactoryMethod = KafkaMessageChannelBinder.class + .getDeclaredMethod("getProducerFactory", String.class, + ExtendedProducerProperties.class, String.class, String.class); + getProducerFactoryMethod.setAccessible(true); + DefaultKafkaProducerFactory producerFactory = (DefaultKafkaProducerFactory) getProducerFactoryMethod + .invoke(this.kafkaMessageChannelBinder, "bar", producerProperties, "bar.producer", "bar"); + Field producerFactoryConfigField = ReflectionUtils + .findField(DefaultKafkaProducerFactory.class, "configs", Map.class); + ReflectionUtils.makeAccessible(producerFactoryConfigField); + Map producerConfigs = (Map) ReflectionUtils + .getField(producerFactoryConfigField, producerFactory); + assertThat(producerConfigs.get("batch.size")).isEqualTo("12345"); + assertThat(producerConfigs.get("linger.ms")).isEqualTo("100"); + assertThat(producerConfigs.get("key.serializer")) + .isEqualTo(ByteArraySerializer.class); + assertThat(producerConfigs.get("value.serializer")) + .isEqualTo(ByteArraySerializer.class); + assertThat(producerConfigs.get("compression.type")).isEqualTo("gzip"); + + Field physicalCloseTimeoutField = ReflectionUtils + .findField(DefaultKafkaProducerFactory.class, "physicalCloseTimeout", Duration.class); + ReflectionUtils.makeAccessible(physicalCloseTimeoutField); + Duration physicalCloseTimeoutConfig = (Duration) ReflectionUtils + .getField(physicalCloseTimeoutField, producerFactory); + assertThat(physicalCloseTimeoutConfig).isEqualTo(Duration.ofSeconds(10)); + + List bootstrapServers = new ArrayList<>(); + bootstrapServers.add("10.98.09.199:9082"); + assertThat((((String) producerConfigs.get("bootstrap.servers")) + .contains("10.98.09.199:9082"))).isTrue(); + Method createKafkaConsumerFactoryMethod = KafkaMessageChannelBinder.class + .getDeclaredMethod("createKafkaConsumerFactory", boolean.class, + String.class, ExtendedConsumerProperties.class, String.class, String.class); + createKafkaConsumerFactoryMethod.setAccessible(true); + ExtendedConsumerProperties consumerProperties = new ExtendedConsumerProperties<>( + new KafkaConsumerProperties()); + DefaultKafkaConsumerFactory consumerFactory = (DefaultKafkaConsumerFactory) createKafkaConsumerFactoryMethod + .invoke(this.kafkaMessageChannelBinder, true, "test", consumerProperties, "test.consumer", "test"); + Field consumerFactoryConfigField = ReflectionUtils + .findField(DefaultKafkaConsumerFactory.class, "configs", Map.class); + ReflectionUtils.makeAccessible(consumerFactoryConfigField); + Map consumerConfigs = (Map) ReflectionUtils + .getField(consumerFactoryConfigField, consumerFactory); + assertThat(consumerConfigs.get("key.deserializer")) + .isEqualTo(ByteArrayDeserializer.class); + assertThat(consumerConfigs.get("value.deserializer")) + .isEqualTo(ByteArrayDeserializer.class); + assertThat((((String) consumerConfigs.get("bootstrap.servers")) + .contains("10.98.09.199:9082"))).isTrue(); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderConfigurationTest.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderConfigurationTest.java new file mode 100644 index 000000000..5770ea646 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderConfigurationTest.java @@ -0,0 +1,57 @@ +/* + * Copyright 2016-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.lang.reflect.Field; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfiguration; +import org.springframework.kafka.support.ProducerListener; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.util.ReflectionUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Ilayaperumal Gopinathan + */ +@RunWith(SpringJUnit4ClassRunner.class) +@SpringBootTest(classes = { KafkaBinderConfiguration.class, KafkaAutoConfiguration.class, + KafkaBinderConfigurationTest.class }) +public class KafkaBinderConfigurationTest { + + @Autowired + private KafkaMessageChannelBinder kafkaMessageChannelBinder; + + @Test + public void testKafkaBinderProducerListener() { + assertThat(this.kafkaMessageChannelBinder).isNotNull(); + Field producerListenerField = ReflectionUtils.findField( + KafkaMessageChannelBinder.class, "producerListener", + ProducerListener.class); + ReflectionUtils.makeAccessible(producerListenerField); + ProducerListener producerListener = (ProducerListener) ReflectionUtils + .getField(producerListenerField, this.kafkaMessageChannelBinder); + assertThat(producerListener).isNotNull(); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicatorTest.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicatorTest.java new file mode 100644 index 000000000..64d84e842 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicatorTest.java @@ -0,0 +1,281 @@ +/* + * Copyright 2017-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.PartitionInfo; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.Status; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.listener.AbstractMessageListenerContainer; + +import static java.util.Collections.singleton; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Barry Commins + * @author Gary Russell + * @author Laur Aliste + * @author Soby Chacko + * @author Chukwubuikem Ume-Ugwa + * @author Taras Danylchuk + */ +public class KafkaBinderHealthIndicatorTest { + + private static final String TEST_TOPIC = "test"; + + private static final String REGEX_TOPIC = "regex*"; + + private KafkaBinderHealthIndicator indicator; + + @Mock + private DefaultKafkaConsumerFactory consumerFactory; + + @Mock + private KafkaConsumer consumer; + + @Mock + AbstractMessageListenerContainer listenerContainerA; + + @Mock + AbstractMessageListenerContainer listenerContainerB; + + @Mock + private KafkaMessageChannelBinder binder; + + private final Map topicsInUse = new HashMap<>(); + + @Before + public void setup() { + MockitoAnnotations.initMocks(this); + org.mockito.BDDMockito.given(consumerFactory.createConsumer()) + .willReturn(consumer); + org.mockito.BDDMockito.given(binder.getTopicsInUse()).willReturn(topicsInUse); + this.indicator = new KafkaBinderHealthIndicator(binder, consumerFactory); + this.indicator.setTimeout(10); + } + + @Test + public void kafkaBinderIsUpWithNoConsumers() { + final List partitions = partitions(new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation( + "group1-healthIndicator", partitions, false)); + org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)) + .willReturn(partitions); + org.mockito.BDDMockito.given(binder.getKafkaMessageListenerContainers()) + .willReturn(Collections.emptyList()); + + Health health = indicator.health(); + assertThat(health.getStatus()).isEqualTo(Status.UP); + assertThat(health.getDetails()).containsEntry("topicsInUse", singleton(TEST_TOPIC)); + } + + @Test + public void kafkaBinderIsUp() { + final List partitions = partitions(new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation( + "group1-healthIndicator", partitions, false)); + org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)) + .willReturn(partitions); + org.mockito.BDDMockito.given(binder.getKafkaMessageListenerContainers()) + .willReturn(Arrays.asList(listenerContainerA, listenerContainerB)); + mockContainer(listenerContainerA, true, true); + mockContainer(listenerContainerB, true, true); + + Health health = indicator.health(); + assertThat(health.getStatus()).isEqualTo(Status.UP); + assertThat(health.getDetails()).containsEntry("topicsInUse", singleton(TEST_TOPIC)); + assertThat(health.getDetails()).hasEntrySatisfying("listenerContainers", value -> + assertThat((ArrayList) value).hasSize(2)); + } + + @Test + public void kafkaBinderIsDownWhenOneOfConsumersIsNotRunning() { + final List partitions = partitions(new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation( + "group1-healthIndicator", partitions, false)); + org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)) + .willReturn(partitions); + org.mockito.BDDMockito.given(binder.getKafkaMessageListenerContainers()) + .willReturn(Arrays.asList(listenerContainerA, listenerContainerB)); + mockContainer(listenerContainerA, false, true); + mockContainer(listenerContainerB, true, true); + + Health health = indicator.health(); + assertThat(health.getStatus()).isEqualTo(Status.UP); + assertThat(health.getDetails()).containsEntry("topicsInUse", singleton(TEST_TOPIC)); + assertThat(health.getDetails()).hasEntrySatisfying("listenerContainers", value -> + assertThat((ArrayList) value).hasSize(2)); + } + + @Test + public void kafkaBinderIsDownWhenOneOfContainersWasStoppedAbnormally() { + final List partitions = partitions(new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation( + "group1-healthIndicator", partitions, false)); + org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)) + .willReturn(partitions); + org.mockito.BDDMockito.given(binder.getKafkaMessageListenerContainers()) + .willReturn(Arrays.asList(listenerContainerA, listenerContainerB)); + mockContainer(listenerContainerA, false, false); + mockContainer(listenerContainerB, true, true); + + Health health = indicator.health(); + assertThat(health.getStatus()).isEqualTo(Status.DOWN); + assertThat(health.getDetails()).containsEntry("topicsInUse", singleton(TEST_TOPIC)); + assertThat(health.getDetails()).hasEntrySatisfying("listenerContainers", value -> + assertThat((ArrayList) value).hasSize(2)); + } + + private void mockContainer(AbstractMessageListenerContainer container, boolean isRunning, + boolean normalState) { + + org.mockito.BDDMockito.given(container.isRunning()).willReturn(isRunning); + org.mockito.BDDMockito.given(container.isContainerPaused()).willReturn(true); + org.mockito.BDDMockito.given(container.getListenerId()).willReturn("someListenerId"); + org.mockito.BDDMockito.given(container.getGroupId()).willReturn("someGroupId"); + org.mockito.BDDMockito.given(container.isInExpectedState()).willReturn(normalState); + } + + @Test + public void kafkaBinderIsUpWithRegexTopic() { + topicsInUse.put(REGEX_TOPIC, new KafkaMessageChannelBinder.TopicInformation( + "regex-healthIndicator", null, true)); + Health health = indicator.health(); + // verify no consumer interaction for retrieving partitions + org.mockito.BDDMockito.verify(consumer, Mockito.never()) + .partitionsFor(REGEX_TOPIC); + // Ensuring the normal health check returns with status "up" + assertThat(health.getStatus()).isEqualTo(Status.UP); + } + + @Test + public void kafkaBinderIsDown() { + final List partitions = partitions(null); + topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation( + "group2-healthIndicator", partitions, false)); + org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)) + .willReturn(partitions); + Health health = indicator.health(); + assertThat(health.getStatus()).isEqualTo(Status.DOWN); + } + + @Test + public void kafkaBinderIsDownWhenConsiderDownWhenAnyPartitionHasNoLeaderIsTrue() { + final List partitions = partitions(new Node(0, null, 0)); + partitions.add(new PartitionInfo(TEST_TOPIC, 0, null, null, null)); + indicator.setConsiderDownWhenAnyPartitionHasNoLeader(true); + topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation( + "group2-healthIndicator", partitions, false)); + org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)) + .willReturn(partitions); + Health health = indicator.health(); + assertThat(health.getStatus()).isEqualTo(Status.DOWN); + } + + @Test + public void kafkaBinderIsUpWhenConsiderDownWhenAnyPartitionHasNoLeaderIsFalse() { + Node node = new Node(0, null, 0); + final List partitions = partitions(node); + partitions.add(new PartitionInfo(TEST_TOPIC, 0, null, null, null)); + indicator.setConsiderDownWhenAnyPartitionHasNoLeader(false); + topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation( + "group2-healthIndicator", partitions(node), false)); + org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)) + .willReturn(partitions); + Health health = indicator.health(); + assertThat(health.getStatus()).isEqualTo(Status.UP); + } + + @Test(timeout = 5000) + public void kafkaBinderDoesNotAnswer() { + final List partitions = partitions(new Node(-1, null, 0)); + topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation( + "group3-healthIndicator", partitions, false)); + org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)) + .willAnswer(invocation -> { + final int fiveMinutes = 1000 * 60 * 5; + Thread.sleep(fiveMinutes); + return partitions; + }); + this.indicator.setTimeout(1); + Health health = indicator.health(); + assertThat(health.getStatus()).isEqualTo(Status.DOWN); + } + + @Test + public void createsConsumerOnceWhenInvokedMultipleTimes() { + final List partitions = partitions(new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation( + "group4-healthIndicator", partitions, false)); + org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)) + .willReturn(partitions); + + indicator.health(); + Health health = indicator.health(); + + assertThat(health.getStatus()).isEqualTo(Status.UP); + org.mockito.Mockito.verify(this.consumerFactory).createConsumer(); + } + + @Test + public void consumerCreationFailsFirstTime() { + final List partitions = partitions(new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation( + "foo-healthIndicator", partitions, false)); + + org.mockito.BDDMockito.given(consumerFactory.createConsumer()) + .willThrow(KafkaException.class).willReturn(consumer); + + Health health = indicator.health(); + assertThat(health.getStatus()).isEqualTo(Status.DOWN); + + health = indicator.health(); + assertThat(health.getStatus()).isEqualTo(Status.UP); + + org.mockito.Mockito.verify(this.consumerFactory, Mockito.times(2)) + .createConsumer(); + } + + @Test + public void testIfNoTopicsRegisteredByTheBinderProvidesDownStatus() { + Health health = indicator.health(); + assertThat(health.getStatus()).isEqualTo(Status.UNKNOWN); + } + + private List partitions(Node leader) { + List partitions = new ArrayList<>(); + partitions.add(new PartitionInfo(TEST_TOPIC, 0, leader, null, null)); + return partitions; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderJaasInitializerListenerTest.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderJaasInitializerListenerTest.java new file mode 100644 index 000000000..a07e3ddf3 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderJaasInitializerListenerTest.java @@ -0,0 +1,134 @@ +/* + * Copyright 2016-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import javax.security.auth.login.AppConfigurationEntry; + +import com.sun.security.auth.login.ConfigFile; +import org.apache.kafka.common.security.JaasUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfiguration; +import org.springframework.core.io.ClassPathResource; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Marius Bogoevici + * @author Soby Chacko + */ +public class KafkaBinderJaasInitializerListenerTest { + + private static final String KAFKA_BROKERS_PROPERTY = "spring.cloud.stream.kafka.binder.brokers"; + + @ClassRule + public static EmbeddedKafkaRule kafkaEmbedded = new EmbeddedKafkaRule(1, true); + + private static String JAVA_LOGIN_CONFIG_PARAM_VALUE; + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withUserConfiguration(KafkaBinderConfiguration.class, KafkaAutoConfiguration.class); + + @BeforeClass + public static void setup() { + System.setProperty(KAFKA_BROKERS_PROPERTY, + kafkaEmbedded.getEmbeddedKafka().getBrokersAsString()); + //Retrieve the current value for this system property if there is one set. + JAVA_LOGIN_CONFIG_PARAM_VALUE = System.getProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM); + System.clearProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM); + } + + @AfterClass + public static void clean() { + System.clearProperty(KAFKA_BROKERS_PROPERTY); + //If there was a previous value for this property, then restore it. + if (JAVA_LOGIN_CONFIG_PARAM_VALUE != null) { + System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, JAVA_LOGIN_CONFIG_PARAM_VALUE); + } + } + + @Before + public void before() { + System.clearProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM); + } + + @Test + public void testConfigurationParsedCorrectlyWithKafkaClientAndDefaultControlFlag() + throws Exception { + ConfigFile configFile = new ConfigFile( + new ClassPathResource("jaas-sample-kafka-only.conf").getURI()); + final AppConfigurationEntry[] kafkaConfigurationArray = configFile + .getAppConfigurationEntry("KafkaClient"); + + + this.contextRunner + .withPropertyValues("spring.cloud.stream.kafka.binder.jaas.options.useKeyTab=true", + "spring.cloud.stream.kafka.binder.jaas.options.storeKey=true", + "spring.cloud.stream.kafka.binder.jaas.options.keyTab=/etc/security/keytabs/kafka_client.keytab", + "spring.cloud.stream.kafka.binder.jaas.options.principal=kafka-client-1@EXAMPLE.COM", + "spring.jmx.enabled=false") + .run(context -> { + javax.security.auth.login.Configuration configuration = javax.security.auth.login.Configuration + .getConfiguration(); + + final AppConfigurationEntry[] kafkaConfiguration = configuration + .getAppConfigurationEntry("KafkaClient"); + assertThat(kafkaConfiguration).hasSize(1); + assertThat(kafkaConfiguration[0].getOptions()) + .isEqualTo(kafkaConfigurationArray[0].getOptions()); + assertThat(kafkaConfiguration[0].getControlFlag()) + .isEqualTo(AppConfigurationEntry.LoginModuleControlFlag.REQUIRED); + }); + } + + @Test + public void testConfigurationParsedCorrectlyWithKafkaClientAndNonDefaultControlFlag() + throws Exception { + ConfigFile configFile = new ConfigFile( + new ClassPathResource("jaas-sample-kafka-only.conf").getURI()); + final AppConfigurationEntry[] kafkaConfigurationArray = configFile + .getAppConfigurationEntry("KafkaClient"); + + this.contextRunner + .withPropertyValues("spring.cloud.stream.kafka.binder.jaas.options.useKeyTab=true", + "spring.cloud.stream.kafka.binder.jaas.controlFlag=requisite", + "spring.cloud.stream.kafka.binder.jaas.options.storeKey=true", + "spring.cloud.stream.kafka.binder.jaas.options.keyTab=/etc/security/keytabs/kafka_client.keytab", + "spring.cloud.stream.kafka.binder.jaas.options.principal=kafka-client-1@EXAMPLE.COM", + "spring.jmx.enabled=false") + .run(context -> { + javax.security.auth.login.Configuration configuration = javax.security.auth.login.Configuration + .getConfiguration(); + + final AppConfigurationEntry[] kafkaConfiguration = configuration + .getAppConfigurationEntry("KafkaClient"); + assertThat(kafkaConfiguration).hasSize(1); + assertThat(kafkaConfiguration[0].getOptions()) + .isEqualTo(kafkaConfigurationArray[0].getOptions()); + assertThat(kafkaConfiguration[0].getControlFlag()) + .isEqualTo(AppConfigurationEntry.LoginModuleControlFlag.REQUISITE); + context.close(); + }); + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderMetricsTest.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderMetricsTest.java new file mode 100644 index 000000000..a1e3a432c --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderMetricsTest.java @@ -0,0 +1,265 @@ +/* + * Copyright 2016-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.config.MeterFilter; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder.TopicInformation; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +/** + * @author Henryk Konsek + * @author Thomas Cheyney + * @author Soby Chacko + */ +public class KafkaBinderMetricsTest { + + private static final String TEST_TOPIC = "test"; + + private KafkaBinderMetrics metrics; + + @Mock + private DefaultKafkaConsumerFactory consumerFactory; + + @Mock + private KafkaConsumer consumer; + + @Mock + private KafkaMessageChannelBinder binder; + + private MeterRegistry meterRegistry = new SimpleMeterRegistry(); + + private Map topicsInUse = new HashMap<>(); + + @Mock + private KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties; + + @Before + public void setup() { + MockitoAnnotations.openMocks(this); + org.mockito.BDDMockito.given(consumerFactory + .createConsumer(ArgumentMatchers.any(), ArgumentMatchers.any())) + .willReturn(consumer); + org.mockito.BDDMockito.given(binder.getTopicsInUse()).willReturn(topicsInUse); + metrics = new KafkaBinderMetrics(binder, kafkaBinderConfigurationProperties, + consumerFactory, null); + org.mockito.BDDMockito + .given(consumer.endOffsets(ArgumentMatchers.anyCollection())) + .willReturn(java.util.Collections + .singletonMap(new TopicPartition(TEST_TOPIC, 0), 1000L)); + } + + @Test + public void shouldIndicateLag() { + final Map committed = new HashMap<>(); + TopicPartition topicPartition = new TopicPartition(TEST_TOPIC, 0); + committed.put(topicPartition, new OffsetAndMetadata(500)); + org.mockito.BDDMockito + .given(consumer.committed(ArgumentMatchers.anySet())) + .willReturn(committed); + List partitions = partitions(new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, + new TopicInformation("group1-metrics", partitions, false)); + org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)) + .willReturn(partitions); + metrics.bindTo(meterRegistry); + assertThat(meterRegistry.getMeters()).hasSize(1); + assertThat(meterRegistry.get(KafkaBinderMetrics.OFFSET_LAG_METRIC_NAME) + .tag("group", "group1-metrics").tag("topic", TEST_TOPIC).gauge().value()) + .isEqualTo(500.0); + } + + @Test + public void shouldNotContainAnyMetricsWhenUsingNoopGauge() { + // Adding NoopGauge for the offset metric. + meterRegistry.config().meterFilter( + MeterFilter.denyNameStartsWith("spring.cloud.stream.binder.kafka.offset")); + + // Because we have NoopGauge for the offset metric in the meter registry, none of these expectations matter. + org.mockito.BDDMockito + .given(consumer.committed(ArgumentMatchers.any(TopicPartition.class))) + .willReturn(new OffsetAndMetadata(500)); + List partitions = partitions(new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, + new TopicInformation("group1-metrics", partitions, false)); + org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)) + .willReturn(partitions); + metrics.bindTo(meterRegistry); + + // Because of the NoopGauge, the meterRegistry should contain no metric. + assertThat(meterRegistry.getMeters()).hasSize(0); + } + + @Test + public void shouldSumUpPartitionsLags() { + Map endOffsets = new HashMap<>(); + endOffsets.put(new TopicPartition(TEST_TOPIC, 0), 1000L); + endOffsets.put(new TopicPartition(TEST_TOPIC, 1), 1000L); + org.mockito.BDDMockito + .given(consumer.endOffsets(ArgumentMatchers.anyCollection())) + .willReturn(endOffsets); + final Map committed = new HashMap<>(); + TopicPartition topicPartition1 = new TopicPartition(TEST_TOPIC, 0); + TopicPartition topicPartition2 = new TopicPartition(TEST_TOPIC, 1); + committed.put(topicPartition1, new OffsetAndMetadata(500)); + committed.put(topicPartition2, new OffsetAndMetadata(500)); + org.mockito.BDDMockito + .given(consumer.committed(ArgumentMatchers.anySet())) + .willReturn(committed); + List partitions = partitions(new Node(0, null, 0), + new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, + new TopicInformation("group2-metrics", partitions, false)); + org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)) + .willReturn(partitions); + metrics.bindTo(meterRegistry); + assertThat(meterRegistry.getMeters()).hasSize(1); + assertThat(meterRegistry.get(KafkaBinderMetrics.OFFSET_LAG_METRIC_NAME) + .tag("group", "group2-metrics").tag("topic", TEST_TOPIC).gauge().value()) + .isEqualTo(1000.0); + } + + @Test + public void shouldIndicateFullLagForNotCommittedGroups() { + List partitions = partitions(new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, + new TopicInformation("group3-metrics", partitions, false)); + org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)) + .willReturn(partitions); + metrics.bindTo(meterRegistry); + assertThat(meterRegistry.getMeters()).hasSize(1); + assertThat(meterRegistry.get(KafkaBinderMetrics.OFFSET_LAG_METRIC_NAME) + .tag("group", "group3-metrics").tag("topic", TEST_TOPIC).gauge().value()) + .isEqualTo(1000.0); + } + + @Test + public void shouldNotCalculateLagForProducerTopics() { + List partitions = partitions(new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, new TopicInformation(null, partitions, false)); + metrics.bindTo(meterRegistry); + assertThat(meterRegistry.getMeters()).isEmpty(); + } + + @Test + public void createsConsumerOnceWhenInvokedMultipleTimes() { + final List partitions = partitions(new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, + new TopicInformation("group4-metrics", partitions, false)); + + metrics.bindTo(meterRegistry); + + Gauge gauge = meterRegistry.get(KafkaBinderMetrics.OFFSET_LAG_METRIC_NAME) + .tag("group", "group4-metrics").tag("topic", TEST_TOPIC).gauge(); + gauge.value(); + assertThat(gauge.value()).isEqualTo(1000.0); + + org.mockito.Mockito.verify(this.consumerFactory) + .createConsumer(ArgumentMatchers.any(), ArgumentMatchers.any()); + } + + @Test + public void consumerCreationFailsFirstTime() { + org.mockito.BDDMockito + .given(consumerFactory.createConsumer(ArgumentMatchers.any(), + ArgumentMatchers.any())) + .willThrow(KafkaException.class).willReturn(consumer); + + final List partitions = partitions(new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, + new TopicInformation("group5-metrics", partitions, false)); + + metrics.bindTo(meterRegistry); + + Gauge gauge = meterRegistry.get(KafkaBinderMetrics.OFFSET_LAG_METRIC_NAME) + .tag("group", "group5-metrics").tag("topic", TEST_TOPIC).gauge(); + assertThat(gauge.value()).isEqualTo(0); + assertThat(gauge.value()).isEqualTo(1000.0); + + org.mockito.Mockito.verify(this.consumerFactory, Mockito.times(2)) + .createConsumer(ArgumentMatchers.any(), ArgumentMatchers.any()); + } + + @Test + public void createOneConsumerPerGroup() { + final List partitions1 = partitions(new Node(0, null, 0)); + final List partitions2 = partitions(new Node(0, null, 0)); + topicsInUse.put(TEST_TOPIC, + new TopicInformation("group1-metrics", partitions1, false)); + topicsInUse.put("test2", + new TopicInformation("group2-metrics", partitions2, false)); + + metrics.bindTo(meterRegistry); + + KafkaConsumer consumer2 = mock(KafkaConsumer.class); + org.mockito.BDDMockito + .given(consumerFactory.createConsumer( + ArgumentMatchers.eq("group2-metrics"), ArgumentMatchers.any())) + .willReturn(consumer2); + org.mockito.BDDMockito + .given(consumer2.endOffsets(ArgumentMatchers.anyCollection())) + .willReturn(java.util.Collections + .singletonMap(new TopicPartition("test2", 0), 50L)); + + Gauge gauge1 = meterRegistry.get(KafkaBinderMetrics.OFFSET_LAG_METRIC_NAME) + .tag("group", "group1-metrics").tag("topic", TEST_TOPIC).gauge(); + Gauge gauge2 = meterRegistry.get(KafkaBinderMetrics.OFFSET_LAG_METRIC_NAME) + .tag("group", "group2-metrics").tag("topic", "test2").gauge(); + gauge1.value(); + gauge2.value(); + assertThat(gauge1.value()).isEqualTo(1000.0); + assertThat(gauge2.value()).isEqualTo(50.0); + + org.mockito.Mockito.verify(this.consumerFactory).createConsumer( + ArgumentMatchers.eq("group1-metrics"), ArgumentMatchers.any()); + org.mockito.Mockito.verify(this.consumerFactory).createConsumer( + ArgumentMatchers.eq("group2-metrics"), ArgumentMatchers.any()); + } + + private List partitions(Node... nodes) { + List partitions = new ArrayList<>(); + for (int i = 0; i < nodes.length; i++) { + partitions.add(new PartitionInfo(TEST_TOPIC, i, nodes[i], null, null)); + } + return partitions; + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderTests.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderTests.java new file mode 100644 index 000000000..667aeb80b --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderTests.java @@ -0,0 +1,3959 @@ +/* + * Copyright 2016-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.IntStream; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.DescribeTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.TopicExistsException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.record.TimestampType; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.LongDeserializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.assertj.core.api.Assertions; +import org.assertj.core.api.Condition; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.cloud.stream.binder.Binder; +import org.springframework.cloud.stream.binder.BinderException; +import org.springframework.cloud.stream.binder.BinderHeaders; +import org.springframework.cloud.stream.binder.Binding; +import org.springframework.cloud.stream.binder.DefaultPollableMessageSource; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.HeaderMode; +import org.springframework.cloud.stream.binder.PartitionCapableBinderTests; +import org.springframework.cloud.stream.binder.PartitionTestSupport; +import org.springframework.cloud.stream.binder.PollableSource; +import org.springframework.cloud.stream.binder.RequeueCurrentMessageException; +import org.springframework.cloud.stream.binder.Spy; +import org.springframework.cloud.stream.binder.TestUtils; +import org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder.TopicInformation; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.cloud.stream.binder.kafka.utils.DlqDestinationResolver; +import org.springframework.cloud.stream.binder.kafka.utils.DlqPartitionFunction; +import org.springframework.cloud.stream.binder.kafka.utils.KafkaTopicUtils; +import org.springframework.cloud.stream.binding.MessageConverterConfigurer.PartitioningInterceptor; +import org.springframework.cloud.stream.config.BindingProperties; +import org.springframework.cloud.stream.provisioning.ProvisioningException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.expression.Expression; +import org.springframework.expression.common.LiteralExpression; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.integration.IntegrationMessageHeaderAccessor; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.channel.QueueChannel; +import org.springframework.integration.context.IntegrationContextUtils; +import org.springframework.integration.handler.BridgeHandler; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler; +import org.springframework.integration.kafka.support.KafkaSendFailureException; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.AbstractMessageListenerContainer; +import org.springframework.kafka.listener.CommonErrorHandler; +import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; +import org.springframework.kafka.listener.ContainerProperties; +import org.springframework.kafka.listener.DefaultErrorHandler; +import org.springframework.kafka.listener.MessageListenerContainer; +import org.springframework.kafka.support.Acknowledgment; +import org.springframework.kafka.support.KafkaHeaderMapper; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.kafka.support.SendResult; +import org.springframework.kafka.support.TopicPartitionOffset; +import org.springframework.kafka.support.converter.BatchMessagingMessageConverter; +import org.springframework.kafka.support.converter.MessagingMessageConverter; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.condition.EmbeddedKafkaCondition; +import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.kafka.test.core.BrokerAddress; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessageHandlingException; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.MessagingException; +import org.springframework.messaging.SubscribableChannel; +import org.springframework.messaging.support.ChannelInterceptor; +import org.springframework.messaging.support.ErrorMessage; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.util.Assert; +import org.springframework.util.MimeTypeUtils; +import org.springframework.util.backoff.FixedBackOff; +import org.springframework.util.concurrent.ListenableFuture; +import org.springframework.util.concurrent.SettableListenableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.Assertions.fail; +import static org.mockito.Mockito.mock; + +/** + * @author Soby Chacko + * @author Ilayaperumal Gopinathan + * @author Henryk Konsek + * @author Gary Russell + */ +@EmbeddedKafka(count = 1, controlledShutdown = true, topics = "error.pollableDlq.group-pcWithDlq", brokerProperties = {"transaction.state.log.replication.factor=1", + "transaction.state.log.min.isr=1"}) +public class KafkaBinderTests extends + + PartitionCapableBinderTests, ExtendedProducerProperties> { + + private static final int DEFAULT_OPERATION_TIMEOUT = 30; + + private final String CLASS_UNDER_TEST_NAME = KafkaMessageChannelBinder.class + .getSimpleName(); + + private KafkaTestBinder binder; + + private AdminClient adminClient; + + private static EmbeddedKafkaBroker embeddedKafka; + + @BeforeAll + public static void setup() { + embeddedKafka = EmbeddedKafkaCondition.getBroker(); + } + + @Override + protected ExtendedConsumerProperties createConsumerProperties() { + final ExtendedConsumerProperties kafkaConsumerProperties = new ExtendedConsumerProperties<>( + new KafkaConsumerProperties()); + // set the default values that would normally be propagated by Spring Cloud Stream + kafkaConsumerProperties.setInstanceCount(1); + kafkaConsumerProperties.setInstanceIndex(0); + return kafkaConsumerProperties; + } + + private ExtendedProducerProperties createProducerProperties() { + return this.createProducerProperties(null); + } + + @Override + protected ExtendedProducerProperties createProducerProperties(TestInfo testInto) { + ExtendedProducerProperties producerProperties = new ExtendedProducerProperties<>( + new KafkaProducerProperties()); + producerProperties.getExtension().setSync(true); + return producerProperties; + } + + @Override + protected void binderBindUnbindLatency() throws InterruptedException { + Thread.sleep(500); + } + + @Override + protected KafkaTestBinder getBinder() { + if (binder == null) { + KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties(); + KafkaTopicProvisioner kafkaTopicProvisioner = new KafkaTopicProvisioner( + binderConfiguration, new TestKafkaProperties(), null); + try { + kafkaTopicProvisioner.afterPropertiesSet(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + binder = new KafkaTestBinder(binderConfiguration, kafkaTopicProvisioner); + } + return binder; + } + + private KafkaTestBinder getBinder( + KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties) { + + return getBinder(kafkaBinderConfigurationProperties, null, null); + } + + private KafkaTestBinder getBinder( + KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties, + DlqPartitionFunction dlqPartitionFunction, DlqDestinationResolver dlqDestinationResolver) { + + KafkaTopicProvisioner provisioningProvider = new KafkaTopicProvisioner( + kafkaBinderConfigurationProperties, new TestKafkaProperties(), null); + try { + provisioningProvider.afterPropertiesSet(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + return new KafkaTestBinder(kafkaBinderConfigurationProperties, + provisioningProvider, dlqPartitionFunction, dlqDestinationResolver); + } + + private KafkaBinderConfigurationProperties createConfigurationProperties() { + KafkaBinderConfigurationProperties binderConfiguration = new KafkaBinderConfigurationProperties( + new TestKafkaProperties()); + BrokerAddress[] brokerAddresses = embeddedKafka.getBrokerAddresses(); + + List bAddresses = new ArrayList<>(); + for (BrokerAddress bAddress : brokerAddresses) { + bAddresses.add(bAddress.toString()); + } + String[] foo = new String[bAddresses.size()]; + binderConfiguration.setBrokers(bAddresses.toArray(foo)); + return binderConfiguration; + } + + private int partitionSize(String topic) { + return consumerFactory().createConsumer().partitionsFor(topic).size(); + } + + private void invokeCreateTopic(String topic, int partitions, int replicationFactor) + throws Exception { + + NewTopic newTopic = new NewTopic(topic, partitions, (short) replicationFactor); + CreateTopicsResult topics = adminClient + .createTopics(Collections.singletonList(newTopic)); + topics.all().get(DEFAULT_OPERATION_TIMEOUT, TimeUnit.SECONDS); + } + + private String getKafkaOffsetHeaderKey() { + return KafkaHeaders.OFFSET; + } + + @BeforeEach + public void init() { + String multiplier = System.getenv("KAFKA_TIMEOUT_MULTIPLIER"); + if (multiplier != null) { + timeoutMultiplier = Double.parseDouble(multiplier); + } + + BrokerAddress[] brokerAddresses = embeddedKafka.getBrokerAddresses(); + List bAddresses = new ArrayList<>(); + for (BrokerAddress bAddress : brokerAddresses) { + bAddresses.add(bAddress.toString()); + } + String[] foo = new String[bAddresses.size()]; + + Map adminConfigs = new HashMap<>(); + adminConfigs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, + bAddresses.toArray(foo)[0]); + adminClient = AdminClient.create(adminConfigs); + } + + private int invokePartitionSize(String topic) throws Throwable { + + DescribeTopicsResult describeTopicsResult = adminClient + .describeTopics(Collections.singletonList(topic)); + KafkaFuture> all = describeTopicsResult.all(); + Map stringTopicDescriptionMap = all + .get(DEFAULT_OPERATION_TIMEOUT, TimeUnit.SECONDS); + TopicDescription topicDescription = stringTopicDescriptionMap.get(topic); + return topicDescription.partitions().size(); + } + + @Override + protected boolean usesExplicitRouting() { + return false; + } + + @Override + protected String getClassUnderTestName() { + return CLASS_UNDER_TEST_NAME; + } + + @Override + public Spy spyOn(final String name) { + throw new UnsupportedOperationException("'spyOn' is not used by Kafka tests"); + } + + private ConsumerFactory consumerFactory() { + Map props = new HashMap<>(); + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + configurationProperties.getKafkaConnectionString()); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); + props.put(ConsumerConfig.GROUP_ID_CONFIG, "TEST-CONSUMER-GROUP"); + Deserializer valueDecoder = new ByteArrayDeserializer(); + Deserializer keyDecoder = new ByteArrayDeserializer(); + + return new DefaultKafkaConsumerFactory<>(props, keyDecoder, valueDecoder); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Test + public void testDefaultHeaderMapper() throws Exception { + Binder binder = getBinder(); + + BindingProperties producerBindingProperties = createProducerBindingProperties( + createProducerProperties()); + DirectChannel moduleOutputChannel = createBindableChannel("output", + producerBindingProperties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.getExtension() + .setTrustedPackages(new String[] { "org.springframework.cloud.stream.binder.kafka" }); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + Binding producerBinding = binder.bindProducer("bar.0", + moduleOutputChannel, producerBindingProperties.getProducer()); + + Binding consumerBinding = binder.bindConsumer("bar.0", + "testSendAndReceiveNoOriginalContentType", moduleInputChannel, + consumerProperties); + binderBindUnbindLatency(); + + final Pojo pojoHeader = new Pojo("testing"); + Message message = org.springframework.integration.support.MessageBuilder + .withPayload("foo") + .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN) + .setHeader("foo", pojoHeader).build(); + + moduleOutputChannel.send(message); + CountDownLatch latch = new CountDownLatch(1); + AtomicReference> inboundMessageRef = new AtomicReference<>(); + moduleInputChannel.subscribe(message1 -> { + try { + inboundMessageRef.set((Message) message1); + } + finally { + latch.countDown(); + } + }); + Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message"); + + Assertions.assertThat(inboundMessageRef.get()).isNotNull(); + Assertions.assertThat(inboundMessageRef.get().getPayload()) + .isEqualTo("foo".getBytes()); + Assertions + .assertThat(inboundMessageRef.get().getHeaders() + .get(MessageHeaders.CONTENT_TYPE)) + .isEqualTo(MimeTypeUtils.TEXT_PLAIN); + Assertions.assertThat(inboundMessageRef.get().getHeaders().get("foo")) + .isInstanceOf(Pojo.class); + Pojo actual = (Pojo) inboundMessageRef.get().getHeaders().get("foo"); + Assertions.assertThat(actual.field).isEqualTo(pojoHeader.field); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Test + public void testCustomHeaderMapper() throws Exception { + + KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties(); + binderConfiguration.setHeaderMapperBeanName("headerMapper"); + + KafkaTopicProvisioner kafkaTopicProvisioner = new KafkaTopicProvisioner( + binderConfiguration, new TestKafkaProperties(), null); + try { + kafkaTopicProvisioner.afterPropertiesSet(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + KafkaTestBinder binder = new KafkaTestBinder(binderConfiguration, kafkaTopicProvisioner); + ((GenericApplicationContext) binder.getApplicationContext()).registerBean("headerMapper", + KafkaHeaderMapper.class, () -> new KafkaHeaderMapper() { + @Override + public void fromHeaders(MessageHeaders headers, Headers target) { + target.add(new RecordHeader("custom-header", "foobar".getBytes())); + } + + @Override + public void toHeaders(Headers source, Map target) { + if (source.headers("custom-header").iterator().hasNext()) { + target.put("custom-header", source.headers("custom-header").iterator().next().value()); + } + + } + }); + + ExtendedProducerProperties producerProperties = createProducerProperties(); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + Binding producerBinding = binder.bindProducer("bar.0", + moduleOutputChannel, producerProperties); + + Binding consumerBinding = binder.bindConsumer("bar.0", + "testSendAndReceiveNoOriginalContentType", moduleInputChannel, + consumerProperties); + binderBindUnbindLatency(); + + Message message = org.springframework.integration.support.MessageBuilder + .withPayload("foo") + .setHeader("foo", MimeTypeUtils.TEXT_PLAIN).build(); + + moduleOutputChannel.send(message); + CountDownLatch latch = new CountDownLatch(1); + AtomicReference> inboundMessageRef = new AtomicReference<>(); + moduleInputChannel.subscribe(message1 -> { + try { + inboundMessageRef.set((Message) message1); + } + finally { + latch.countDown(); + } + }); + Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message"); + + Assertions.assertThat(inboundMessageRef.get()).isNotNull(); + Assertions.assertThat(inboundMessageRef.get().getPayload()) + .isEqualTo("foo".getBytes()); + Assertions.assertThat(inboundMessageRef.get().getHeaders().get("foo")) + .isNull(); + Assertions.assertThat(inboundMessageRef.get().getHeaders().get("custom-header")) + .isEqualTo("foobar".getBytes()); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Test + public void testWellKnownHeaderMapperWithBeanNameKafkaHeaderMapper() throws Exception { + + KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties(); + + KafkaTopicProvisioner kafkaTopicProvisioner = new KafkaTopicProvisioner( + binderConfiguration, new TestKafkaProperties(), null); + try { + kafkaTopicProvisioner.afterPropertiesSet(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + KafkaTestBinder binder = new KafkaTestBinder(binderConfiguration, kafkaTopicProvisioner); + ((GenericApplicationContext) binder.getApplicationContext()).registerBean("kafkaBinderHeaderMapper", + KafkaHeaderMapper.class, () -> new BinderHeaderMapper() { + @Override + public void fromHeaders(MessageHeaders headers, Headers target) { + target.add(new RecordHeader("custom-header", "foobar".getBytes())); + super.fromHeaders(headers, target); + } + + @Override + public void toHeaders(Headers source, Map target) { + if (source.headers("custom-header").iterator().hasNext()) { + target.put("custom-header", source.headers("custom-header").iterator().next().value()); + } + + } + }); + + ExtendedProducerProperties producerProperties = createProducerProperties(); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + Binding producerBinding = binder.bindProducer("bar.0", + moduleOutputChannel, producerProperties); + + Binding consumerBinding = binder.bindConsumer("bar.0", + "testSendAndReceiveNoOriginalContentType", moduleInputChannel, + consumerProperties); + binderBindUnbindLatency(); + + Message message = org.springframework.integration.support.MessageBuilder + .withPayload("foo") + .setHeader("foo", MimeTypeUtils.TEXT_PLAIN).build(); + + moduleOutputChannel.send(message); + CountDownLatch latch = new CountDownLatch(1); + AtomicReference> inboundMessageRef = new AtomicReference<>(); + moduleInputChannel.subscribe(message1 -> { + try { + inboundMessageRef.set((Message) message1); + } + finally { + latch.countDown(); + } + }); + Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message"); + + Assertions.assertThat(inboundMessageRef.get()).isNotNull(); + Assertions.assertThat(inboundMessageRef.get().getPayload()) + .isEqualTo("foo".getBytes()); + Assertions.assertThat(inboundMessageRef.get().getHeaders().get("foo")) + .isNull(); + Assertions.assertThat(inboundMessageRef.get().getHeaders().get("custom-header")) + .isEqualTo("foobar".getBytes()); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + @Override + @SuppressWarnings("unchecked") + public void testSendAndReceiveNoOriginalContentType(TestInfo testInfo) throws Exception { + Binder binder = getBinder(); + + BindingProperties producerBindingProperties = createProducerBindingProperties( + createProducerProperties()); + DirectChannel moduleOutputChannel = createBindableChannel("output", + producerBindingProperties); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + Binding producerBinding = binder.bindProducer("bar.0", + moduleOutputChannel, producerBindingProperties.getProducer()); + + consumerProperties.getExtension() + .setTrustedPackages(new String[] { "org.springframework.util" }); + Binding consumerBinding = binder.bindConsumer("bar.0", + "testSendAndReceiveNoOriginalContentType", moduleInputChannel, + consumerProperties); + binderBindUnbindLatency(); + + // TODO: Will have to fix the MimeType to convert to byte array once this issue + // has been resolved: + // https://github.com/spring-projects/spring-kafka/issues/424 + Message message = org.springframework.integration.support.MessageBuilder + .withPayload("foo") + .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN).build(); + moduleOutputChannel.send(message); + CountDownLatch latch = new CountDownLatch(1); + AtomicReference> inboundMessageRef = new AtomicReference<>(); + moduleInputChannel.subscribe(message1 -> { + try { + inboundMessageRef.set((Message) message1); + } + finally { + latch.countDown(); + } + }); + Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message"); + + assertThat(inboundMessageRef.get()).isNotNull(); + assertThat(inboundMessageRef.get().getPayload()).isEqualTo("foo".getBytes()); + assertThat(inboundMessageRef.get().getHeaders().get(MessageHeaders.CONTENT_TYPE)) + .isEqualTo(MimeTypeUtils.TEXT_PLAIN); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + @Override + @SuppressWarnings("unchecked") + public void testSendAndReceive(TestInfo testInfo) throws Exception { + Binder binder = getBinder(); + BindingProperties outputBindingProperties = createProducerBindingProperties( + createProducerProperties()); + DirectChannel moduleOutputChannel = createBindableChannel("output", + outputBindingProperties); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + MessagingMessageConverter mmc = new MessagingMessageConverter(); + ((GenericApplicationContext) ((KafkaTestBinder) binder).getApplicationContext()) + .registerBean("tSARmmc", MessagingMessageConverter.class, () -> mmc); + consumerProperties.getExtension().setConverterBeanName("tSARmmc"); + + Binding producerBinding = binder.bindProducer("foo.bar", + moduleOutputChannel, outputBindingProperties.getProducer()); + Binding consumerBinding = binder.bindConsumer("foo.bar", + "testSendAndReceive", moduleInputChannel, consumerProperties); + assertThat(KafkaTestUtils.getPropertyValue(consumerBinding, + "lifecycle.messageListenerContainer.applicationContext")).isNotNull(); + Message message = org.springframework.integration.support.MessageBuilder + .withPayload("foo".getBytes(StandardCharsets.UTF_8)) + .setHeader(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.APPLICATION_OCTET_STREAM) + .build(); + + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + CountDownLatch latch = new CountDownLatch(1); + AtomicReference> inboundMessageRef = new AtomicReference<>(); + moduleInputChannel.subscribe(message1 -> { + try { + inboundMessageRef.set((Message) message1); + } + finally { + latch.countDown(); + } + }); + Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message"); + + assertThat(inboundMessageRef.get()).isNotNull(); + assertThat( + new String(inboundMessageRef.get().getPayload(), StandardCharsets.UTF_8)) + .isEqualTo("foo"); + assertThat(inboundMessageRef.get().getHeaders().get(MessageHeaders.CONTENT_TYPE)) + .isEqualTo(MimeTypeUtils.APPLICATION_OCTET_STREAM); + + Map topicsInUse = ((KafkaTestBinder) binder) + .getCoreBinder().getTopicsInUse(); + assertThat(topicsInUse.keySet()).contains("foo.bar"); + TopicInformation topic = topicsInUse.get("foo.bar"); + assertThat(topic.isConsumerTopic()).isTrue(); + assertThat(topic.getConsumerGroup()).isEqualTo("testSendAndReceive"); + + assertThat(KafkaTestUtils.getPropertyValue(consumerBinding, "lifecycle.recordListener.messageConverter")) + .isSameAs(mmc); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + @SuppressWarnings({ "unchecked", "rawtypes" }) + public void testSendAndReceiveBatch() throws Exception { + Binder binder = getBinder(); + BindingProperties outputBindingProperties = createProducerBindingProperties( + createProducerProperties()); + DirectChannel moduleOutputChannel = createBindableChannel("output", + outputBindingProperties); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setBatchMode(true); + consumerProperties.getExtension().getConfiguration().put("fetch.min.bytes", "1000"); + consumerProperties.getExtension().getConfiguration().put("fetch.max.wait.ms", "5000"); + consumerProperties.getExtension().getConfiguration().put("max.poll.records", "2"); + BatchMessagingMessageConverter bmmc = new BatchMessagingMessageConverter(); + ((GenericApplicationContext) ((KafkaTestBinder) binder).getApplicationContext()) + .registerBean("tSARBbmmc", BatchMessagingMessageConverter.class, () -> bmmc); + consumerProperties.getExtension().setConverterBeanName("tSARBbmmc"); + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + Binding producerBinding = binder.bindProducer("c.batching", + moduleOutputChannel, outputBindingProperties.getProducer()); + Binding consumerBinding = binder.bindConsumer("c.batching", + "testSendAndReceiveBatch", moduleInputChannel, consumerProperties); + Message message = org.springframework.integration.support.MessageBuilder + .withPayload("foo".getBytes(StandardCharsets.UTF_8)) + .setHeader(KafkaHeaders.PARTITION_ID, 0) + .build(); + + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + message = MessageBuilder + .withPayload("bar".getBytes(StandardCharsets.UTF_8)) + .setHeader(KafkaHeaders.PARTITION_ID, 0) + .build(); + moduleOutputChannel.send(message); + CountDownLatch latch = new CountDownLatch(1); + AtomicReference>> inboundMessageRef = new AtomicReference<>(); + moduleInputChannel.subscribe(message1 -> { + try { + inboundMessageRef.compareAndSet(null, (Message>) message1); + } + finally { + latch.countDown(); + } + }); + Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message"); + + assertThat(inboundMessageRef.get()).isNotNull(); + List payload = inboundMessageRef.get().getPayload(); + assertThat(payload.get(0)).isEqualTo("foo".getBytes()); + if (payload.size() > 1) { // it's a race as to whether we'll get them both or just one. + assertThat(payload.get(1)).isEqualTo("bar".getBytes()); + } + + assertThat(KafkaTestUtils.getPropertyValue(consumerBinding, "lifecycle.batchListener.batchMessageConverter")) + .isSameAs(bmmc); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testDlqWithNativeSerializationEnabledOnDlqProducer() throws Exception { + Binder binder = getBinder(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + + // Native serialization for producer + producerProperties.setUseNativeEncoding(true); + Map producerConfig = new HashMap<>(); + producerConfig.put("value.serializer", + "org.apache.kafka.common.serialization.StringSerializer"); + producerProperties.getExtension().setConfiguration(producerConfig); + + BindingProperties outputBindingProperties = createProducerBindingProperties( + producerProperties); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + outputBindingProperties); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + + // Native Deserialization for consumer + consumerProperties.setUseNativeDecoding(true); + Map consumerConfig = new HashMap<>(); + consumerConfig.put("value.deserializer", + "org.apache.kafka.common.serialization.StringDeserializer"); + consumerProperties.getExtension().setConfiguration(consumerConfig); + + // Setting dlq producer properties on the consumer + consumerProperties.getExtension() + .setDlqProducerProperties(producerProperties.getExtension()); + consumerProperties.getExtension().setEnableDlq(true); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + Binding producerBinding = binder.bindProducer("foo.bar", + moduleOutputChannel, outputBindingProperties.getProducer()); + Binding consumerBinding = binder.bindConsumer("foo.bar", + "testDlqWithNativeEncoding-1", moduleInputChannel, consumerProperties); + + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + + FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler(); + moduleInputChannel.subscribe(handler); + + // Consumer for the DLQ destination + QueueChannel dlqChannel = new QueueChannel(); + ExtendedConsumerProperties dlqConsumerProperties = createConsumerProperties(); + dlqConsumerProperties.setMaxAttempts(1); + + Binding dlqConsumerBinding = binder.bindConsumer( + "error.foo.bar." + "testDlqWithNativeEncoding-1", null, dlqChannel, + dlqConsumerProperties); + binderBindUnbindLatency(); + + Message message = org.springframework.integration.support.MessageBuilder + .withPayload("foo").build(); + + moduleOutputChannel.send(message); + Message receivedMessage = receive(dlqChannel, 5); + assertThat(receivedMessage).isNotNull(); + assertThat(receivedMessage.getPayload()).isEqualTo("foo".getBytes()); + Awaitility.await().until(() -> handler.getInvocationCount() == consumerProperties.getMaxAttempts()); + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_ORIGINAL_TOPIC)) + .isEqualTo("foo.bar".getBytes(StandardCharsets.UTF_8)); + assertThat(new String((byte[]) receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_EXCEPTION_MESSAGE))).startsWith( + "Dispatcher failed to deliver Message; nested exception is java.lang.RuntimeException: fail"); + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_EXCEPTION_STACKTRACE)).isNotNull(); + binderBindUnbindLatency(); + + dlqConsumerBinding.unbind(); + + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testDlqWithNativeDecodingOnConsumerButMissingSerializerOnDlqProducer() + throws Exception { + Binder binder = getBinder(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + // Native serialization for producer + producerProperties.setUseNativeEncoding(true); + Map producerConfig = new HashMap<>(); + producerConfig.put("value.serializer", + "org.apache.kafka.common.serialization.StringSerializer"); + producerProperties.getExtension().setConfiguration(producerConfig); + BindingProperties outputBindingProperties = createProducerBindingProperties( + producerProperties); + DirectChannel moduleOutputChannel = createBindableChannel("output", + outputBindingProperties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + // Native Deserialization for consumer + consumerProperties.setUseNativeDecoding(true); + + Map consumerConfig = new HashMap<>(); + consumerConfig.put("value.deserializer", + "org.apache.kafka.common.serialization.StringDeserializer"); + + // No Dlq producer properties set on the consumer with a native serializer. + // This should cause an error for DLQ sending. + + consumerProperties.getExtension().setConfiguration(consumerConfig); + consumerProperties.getExtension().setEnableDlq(true); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + Binding producerBinding = binder.bindProducer("foo.bar", + moduleOutputChannel, outputBindingProperties.getProducer()); + Binding consumerBinding = binder.bindConsumer("foo.bar", + "testDlqWithNativeEncoding-2", moduleInputChannel, consumerProperties); + + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + + FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler(); + moduleInputChannel.subscribe(handler); + + // Consumer for the DLQ destination + QueueChannel dlqChannel = new QueueChannel(); + ExtendedConsumerProperties dlqConsumerProperties = createConsumerProperties(); + dlqConsumerProperties.setMaxAttempts(1); + + Binding dlqConsumerBinding = binder.bindConsumer( + "error.foo.bar." + "testDlqWithNativeEncoding-2", null, dlqChannel, + dlqConsumerProperties); + binderBindUnbindLatency(); + + Message message = org.springframework.integration.support.MessageBuilder + .withPayload("foo").build(); + + moduleOutputChannel.send(message); + + Message receivedMessage = dlqChannel.receive(5000); + // Ensure that we didn't receive anything on DLQ because of serializer config + // missing on dlq producer while native Decoding is enabled. + assertThat(receivedMessage).isNull(); + + binderBindUnbindLatency(); + + dlqConsumerBinding.unbind(); + + producerBinding.unbind(); + consumerBinding.unbind(); + } + + // For more details on the context of this test: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/657 + @Test + @SuppressWarnings("unchecked") + public void testDlqWithProducerPropertiesSetAtBinderLevel() + throws Exception { + + KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties(); + + Map consumerProps = new HashMap<>(); + consumerProps.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + binderConfiguration.setConsumerProperties(consumerProps); + + Map producerProps = new HashMap<>(); + producerProps.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + binderConfiguration.setProducerProperties(producerProps); + Binder binder = getBinder(binderConfiguration); + + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setUseNativeEncoding(true); + BindingProperties outputBindingProperties = createProducerBindingProperties( + producerProperties); + DirectChannel moduleOutputChannel = createBindableChannel("output", + outputBindingProperties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setUseNativeDecoding(true); + consumerProperties.getExtension().setEnableDlq(true); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + Binding producerBinding = binder.bindProducer("foo.bar", + moduleOutputChannel, outputBindingProperties.getProducer()); + Binding consumerBinding = binder.bindConsumer("foo.bar", + "tdwcapsabl", moduleInputChannel, consumerProperties); + + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + + FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler(); + moduleInputChannel.subscribe(handler); + + // Consumer for the DLQ destination + QueueChannel dlqChannel = new QueueChannel(); + ExtendedConsumerProperties dlqConsumerProperties = createConsumerProperties(); + dlqConsumerProperties.setMaxAttempts(1); + + Binding dlqConsumerBinding = binder.bindConsumer( + "error.foo.bar." + "tdwcapsabl", null, dlqChannel, + dlqConsumerProperties); + binderBindUnbindLatency(); + + Message message = org.springframework.integration.support.MessageBuilder + .withPayload("foo").build(); + + moduleOutputChannel.send(message); + + Message receivedMessage = dlqChannel.receive(5000); + assertThat(receivedMessage).isNotNull(); + assertThat(receivedMessage.getPayload()).isEqualTo("foo"); + + binderBindUnbindLatency(); + + dlqConsumerBinding.unbind(); + + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + public void testDlqAndRetry() throws Exception { + testDlqGuts(true, null, null, false, false); + } + + @Test + public void testDlqAndRetryTransactional() throws Exception { + testDlqGuts(true, null, null, true, false); + } + + @Test + public void testDlq() throws Exception { + testDlqGuts(false, null, 3, false, false); + } + + @Test + public void testDlqWithDlqDestinationResolver() throws Exception { + testDlqGuts(false, null, 3, false, true); + } + + @Test + public void testDlqTransactional() throws Exception { + testDlqGuts(false, null, 3, true, false); + } + + @Test + public void testDlqNone() throws Exception { + testDlqGuts(false, HeaderMode.none, 1, false, false); + } + + @Test + public void testDlqEmbedded() throws Exception { + testDlqGuts(false, HeaderMode.embeddedHeaders, 3, false, false); + } + + private void testDlqGuts(boolean withRetry, HeaderMode headerMode, Integer dlqPartitions, + boolean transactional, boolean useDlqDestResolver) throws Exception { + + int expectedDlqPartition = dlqPartitions == null ? 0 : dlqPartitions - 1; + KafkaBinderConfigurationProperties binderConfig = createConfigurationProperties(); + if (transactional) { + binderConfig.getTransaction().setTransactionIdPrefix("tx-"); + binderConfig.getTransaction().getProducer().getConfiguration().put(ProducerConfig.RETRIES_CONFIG, "1"); + binderConfig.setRequiredAcks("all"); + } + DlqPartitionFunction dlqPartitionFunction; + if (Integer.valueOf(1).equals(dlqPartitions)) { + dlqPartitionFunction = null; // test that ZERO_PARTITION is used + } + else if (dlqPartitions == null) { + dlqPartitionFunction = (group, rec, ex) -> 0; + } + else { + dlqPartitionFunction = (group, rec, ex) -> dlqPartitions - 1; + } + DlqDestinationResolver dlqDestinationResolver = null; + if (useDlqDestResolver) { + dlqDestinationResolver = (cr, e) -> "foo.dlq"; + } + AbstractKafkaTestBinder binder = getBinder(binderConfig, dlqPartitionFunction, dlqDestinationResolver); + + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.getExtension() + .setHeaderPatterns(new String[] { MessageHeaders.CONTENT_TYPE }); + producerProperties.setHeaderMode(headerMode); + producerProperties.setPartitionCount(2); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + + consumerProperties.setMaxAttempts(withRetry ? 2 : 1); + consumerProperties.setBackOffInitialInterval(100); + consumerProperties.setBackOffMaxInterval(150); + consumerProperties.getExtension().setEnableDlq(true); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + consumerProperties.setHeaderMode(headerMode); + consumerProperties.setMultiplex(true); + consumerProperties.getExtension().setDlqPartitions(dlqPartitions); + consumerProperties.setConcurrency(2); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + QueueChannel dlqChannel = new QueueChannel(); + FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler(); + moduleInputChannel.subscribe(handler); + + long uniqueBindingId = System.currentTimeMillis(); + + String producerName = "dlqTest." + uniqueBindingId + ".0"; + Binding producerBinding = binder.bindProducer(producerName, + moduleOutputChannel, producerProperties); + String consumerDest = producerName + ", " + producerName.replaceAll("0", "1"); + Binding consumerBinding = binder.bindConsumer(consumerDest, + "testGroup", moduleInputChannel, consumerProperties); + + AbstractMessageListenerContainer container = TestUtils.getPropertyValue(consumerBinding, + "lifecycle.messageListenerContainer", AbstractMessageListenerContainer.class); + assertThat(container.getContainerProperties().getTopicPartitions().length) + .isEqualTo(4); // 2 topics 2 partitions each + if (transactional) { + assertThat(TestUtils.getPropertyValue(container.getAfterRollbackProcessor(), "kafkaTemplate")).isNotNull(); + assertThat( + TestUtils.getPropertyValue(container.getAfterRollbackProcessor(), "commitRecovered", Boolean.class)) + .isTrue(); + } + + String dlqTopic = useDlqDestResolver ? "foo.dlq" : "error.dlqTest." + uniqueBindingId + ".0.testGroup"; + try (AdminClient admin = AdminClient.create(Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, + embeddedKafka.getBrokersAsString()))) { + if (useDlqDestResolver) { + List nonProvisionedDlqTopics = new ArrayList<>(); + NewTopic nTopic = new NewTopic(dlqTopic, 3, (short) 1); + nonProvisionedDlqTopics.add(nTopic); + admin.createTopics(nonProvisionedDlqTopics); + } + Map topicDescriptions = admin.describeTopics(Collections.singletonList(dlqTopic)) + .all() + .get(10, TimeUnit.SECONDS); + assertThat(topicDescriptions).hasSize(1); + assertThat(topicDescriptions.values().iterator().next().partitions()) + .hasSize(dlqPartitions == null ? 2 : dlqPartitions); + } + + ExtendedConsumerProperties dlqConsumerProperties = createConsumerProperties(); + dlqConsumerProperties.setMaxAttempts(1); + dlqConsumerProperties.setHeaderMode(headerMode); + dlqConsumerProperties.getExtension().setTxCommitRecovered(false); + + ApplicationContext context = TestUtils.getPropertyValue(binder.getBinder(), + "applicationContext", ApplicationContext.class); + SubscribableChannel boundErrorChannel = context + .getBean(consumerDest + ".testGroup.errors-0", SubscribableChannel.class); + SubscribableChannel globalErrorChannel = context.getBean("errorChannel", + SubscribableChannel.class); + final AtomicReference> boundErrorChannelMessage = new AtomicReference<>(); + final AtomicReference> globalErrorChannelMessage = new AtomicReference<>(); + final AtomicBoolean hasRecovererInCallStack = new AtomicBoolean(!withRetry); + final AtomicBoolean hasAfterRollbackProcessorInStack = new AtomicBoolean(!withRetry); + boundErrorChannel.subscribe(message -> { + boundErrorChannelMessage.set(message); + String stackTrace = Arrays.toString(new RuntimeException().getStackTrace()); + hasRecovererInCallStack + .set(stackTrace.contains("ErrorMessageSendingRecoverer")); + hasAfterRollbackProcessorInStack.set(stackTrace.contains("DefaultAfterRollbackProcessor")); + }); + globalErrorChannel.subscribe(globalErrorChannelMessage::set); + + Binding dlqConsumerBinding = binder.bindConsumer( + dlqTopic, null, dlqChannel, + dlqConsumerProperties); + binderBindUnbindLatency(); + if (transactional) { + assertThat(TestUtils.getPropertyValue(dlqConsumerBinding, + "lifecycle.messageListenerContainer.afterRollbackProcessor.kafkaTemplate")).isNotNull(); + assertThat( + TestUtils.getPropertyValue(dlqConsumerBinding, + "lifecycle.messageListenerContainer.afterRollbackProcessor.commitRecovered", Boolean.class)) + .isFalse(); + } + + String testMessagePayload = "test." + UUID.randomUUID().toString(); + Message testMessage = MessageBuilder + .withPayload(testMessagePayload.getBytes()) + .setHeader(KafkaHeaders.PARTITION_ID, 1) + .build(); + moduleOutputChannel.send(testMessage); + + Message receivedMessage = receive(dlqChannel, 3); + assertThat(receivedMessage).isNotNull(); + assertThat(receivedMessage.getPayload()).isEqualTo(testMessagePayload.getBytes()); + if (HeaderMode.embeddedHeaders.equals(headerMode)) { + assertThat(handler.getInvocationCount()) + .isEqualTo(consumerProperties.getMaxAttempts()); + + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_ORIGINAL_TOPIC)) + .isEqualTo(producerName); + + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_ORIGINAL_PARTITION)).isEqualTo(1); + + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_ORIGINAL_OFFSET)).isEqualTo(0); + + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_ORIGINAL_TIMESTAMP)).isNotNull(); + + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_ORIGINAL_TIMESTAMP_TYPE)) + .isEqualTo(TimestampType.CREATE_TIME.toString()); + + assertThat(((String) receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_EXCEPTION_MESSAGE))).startsWith( + "Dispatcher failed to deliver Message; nested exception " + + "is java.lang.RuntimeException: fail"); + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_EXCEPTION_STACKTRACE)).isNotNull(); + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_EXCEPTION_FQCN)).isNotNull(); + assertThat(receivedMessage.getHeaders() + .get(KafkaHeaders.RECEIVED_PARTITION_ID)).isEqualTo(expectedDlqPartition); + } + else if (!HeaderMode.none.equals(headerMode)) { + assertThat(handler.getInvocationCount()) + .isEqualTo(consumerProperties.getMaxAttempts()); + + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_ORIGINAL_TOPIC)) + .isEqualTo(producerName.getBytes(StandardCharsets.UTF_8)); + + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_ORIGINAL_PARTITION)).isEqualTo( + ByteBuffer.allocate(Integer.BYTES).putInt(1).array()); + + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_ORIGINAL_OFFSET)).isEqualTo( + ByteBuffer.allocate(Long.BYTES).putLong(0).array()); + + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_ORIGINAL_TIMESTAMP)).isNotNull(); + + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_ORIGINAL_TIMESTAMP_TYPE)) + .isEqualTo(TimestampType.CREATE_TIME.toString().getBytes()); + + if (transactional) { + assertThat(new String((byte[]) receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_EXCEPTION_MESSAGE))).startsWith( + "Transaction rollback limit exceeded"); + assertThat(new String((byte[]) receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_EXCEPTION_MESSAGE))).contains( + "Dispatcher failed to deliver Message; nested exception " + + "is java.lang.RuntimeException: fail"); + } + else { + assertThat(new String((byte[]) receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_EXCEPTION_MESSAGE))).startsWith( + "Dispatcher failed to deliver Message; nested exception " + + "is java.lang.RuntimeException: fail"); + } + + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_EXCEPTION_STACKTRACE)).isNotNull(); + + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_EXCEPTION_FQCN)).isNotNull(); + + assertThat(receivedMessage.getHeaders() + .get(KafkaHeaders.RECEIVED_PARTITION_ID)).isEqualTo(expectedDlqPartition); + } + else { + assertThat(receivedMessage.getHeaders() + .get(KafkaMessageChannelBinder.X_ORIGINAL_TOPIC)).isNull(); + } + binderBindUnbindLatency(); + + // verify we got a message on the dedicated error channel and the global (via + // bridge) + assertThat(boundErrorChannelMessage.get()).isNotNull(); + assertThat(globalErrorChannelMessage.get()).isNotNull(); + assertThat(hasRecovererInCallStack.get()).isEqualTo(withRetry && !transactional); + assertThat(hasAfterRollbackProcessorInStack.get()).isEqualTo(transactional); + + dlqConsumerBinding.unbind(); + consumerBinding.unbind(); + producerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testDefaultAutoCommitOnErrorWithDlq() throws Exception { + Binder binder = getBinder(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + BindingProperties producerBindingProperties = createProducerBindingProperties( + producerProperties); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + producerBindingProperties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setMaxAttempts(3); + consumerProperties.setBackOffInitialInterval(100); + consumerProperties.setBackOffMaxInterval(150); + consumerProperties.getExtension().setEnableDlq(true); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler(); + moduleInputChannel.subscribe(handler); + long uniqueBindingId = System.currentTimeMillis(); + Binding producerBinding = binder.bindProducer( + "retryTest." + uniqueBindingId + ".0", moduleOutputChannel, + producerProperties); + Binding consumerBinding = binder.bindConsumer( + "retryTest." + uniqueBindingId + ".0", "testGroup", moduleInputChannel, + consumerProperties); + ExtendedConsumerProperties dlqConsumerProperties = createConsumerProperties(); + dlqConsumerProperties.setMaxAttempts(1); + QueueChannel dlqChannel = new QueueChannel(); + Binding dlqConsumerBinding = binder.bindConsumer( + "error.retryTest." + uniqueBindingId + ".0.testGroup", null, dlqChannel, + dlqConsumerProperties); + + String testMessagePayload = "test." + UUID.randomUUID().toString(); + Message testMessage = MessageBuilder + .withPayload(testMessagePayload.getBytes()).build(); + moduleOutputChannel.send(testMessage); + + Message dlqMessage = receive(dlqChannel, 3); + assertThat(dlqMessage).isNotNull(); + assertThat(dlqMessage.getPayload()).isEqualTo(testMessagePayload.getBytes()); + + // first attempt fails + assertThat(handler.getReceivedMessages().entrySet()).hasSize(1); + Message handledMessage = handler.getReceivedMessages().entrySet().iterator() + .next().getValue(); + assertThat(handledMessage).isNotNull(); + assertThat( + new String((byte[]) handledMessage.getPayload(), StandardCharsets.UTF_8)) + .isEqualTo(testMessagePayload); + assertThat(handler.getInvocationCount()) + .isEqualTo(consumerProperties.getMaxAttempts()); + binderBindUnbindLatency(); + dlqConsumerBinding.unbind(); + consumerBinding.unbind(); + + // on the second attempt the message is not redelivered because the DLQ is set + QueueChannel successfulInputChannel = new QueueChannel(); + consumerBinding = binder.bindConsumer("retryTest." + uniqueBindingId + ".0", + "testGroup", successfulInputChannel, consumerProperties); + String testMessage2Payload = "test." + UUID.randomUUID().toString(); + Message testMessage2 = MessageBuilder + .withPayload(testMessage2Payload.getBytes()).build(); + moduleOutputChannel.send(testMessage2); + + Message receivedMessage = receive(successfulInputChannel); + assertThat(receivedMessage.getPayload()) + .isEqualTo(testMessage2Payload.getBytes()); + + binderBindUnbindLatency(); + consumerBinding.unbind(); + producerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testRetriesWithoutDlq() throws Exception { + Binder binder = getBinder(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + BindingProperties producerBindingProperties = createProducerBindingProperties( + producerProperties); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + producerBindingProperties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setMaxAttempts(2); + consumerProperties.setBackOffInitialInterval(100); + consumerProperties.setBackOffMaxInterval(150); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler(); + moduleInputChannel.subscribe(handler); + long uniqueBindingId = System.currentTimeMillis(); + Binding producerBinding = binder.bindProducer( + "retryTest." + uniqueBindingId + ".0", moduleOutputChannel, + producerProperties); + Binding consumerBinding = binder.bindConsumer( + "retryTest." + uniqueBindingId + ".0", "testGroup", moduleInputChannel, + consumerProperties); + + String testMessagePayload = "test." + UUID.randomUUID(); + Message testMessage = MessageBuilder + .withPayload(testMessagePayload.getBytes()).build(); + moduleOutputChannel.send(testMessage); + + Thread.sleep(3000); + + // Since we don't have a DLQ, assert that we are invoking the handler exactly the same number of times + // as set in consumerProperties.maxAttempt and not the default set by Spring Kafka (10 times). + assertThat(handler.getInvocationCount()) + .isEqualTo(consumerProperties.getMaxAttempts()); + binderBindUnbindLatency(); + consumerBinding.unbind(); + producerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testCommonErrorHandlerBeanNameOnConsumerBinding() throws Exception { + Binder binder = getBinder(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + BindingProperties producerBindingProperties = createProducerBindingProperties( + producerProperties); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + producerBindingProperties); + + CountDownLatch latch = new CountDownLatch(1); + CommonErrorHandler commonErrorHandler = new DefaultErrorHandler(new FixedBackOff(0L, 0L)) { + @Override + public void handleRemaining(Exception thrownException, List> records, + Consumer consumer, MessageListenerContainer container) { + super.handleRemaining(thrownException, records, consumer, container); + latch.countDown(); + } + }; + + ConfigurableApplicationContext context = TestUtils.getPropertyValue(binder, + "binder.applicationContext", ConfigurableApplicationContext.class); + context.getBeanFactory().registerSingleton("fooCommonErrorHandler", commonErrorHandler); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setMaxAttempts(2); + consumerProperties.setBackOffInitialInterval(100); + consumerProperties.setBackOffMaxInterval(150); + consumerProperties.getExtension().setCommonErrorHandlerBeanName("fooCommonErrorHandler"); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler(); + moduleInputChannel.subscribe(handler); + long uniqueBindingId = System.currentTimeMillis(); + Binding producerBinding = binder.bindProducer( + "retryTest." + uniqueBindingId + ".0", moduleOutputChannel, + producerProperties); + Binding consumerBinding = binder.bindConsumer( + "retryTest." + uniqueBindingId + ".0", "testGroup", moduleInputChannel, + consumerProperties); + + String testMessagePayload = "test." + UUID.randomUUID(); + Message testMessage = MessageBuilder + .withPayload(testMessagePayload.getBytes()).build(); + moduleOutputChannel.send(testMessage); + + Thread.sleep(3000); + + //Assertions for the CommonErrorHandler configured on the consumer binding (commonErrorHandlerBeanName). + assertThat(KafkaTestUtils.getPropertyValue(consumerBinding, + "lifecycle.messageListenerContainer.commonErrorHandler")).isSameAs(commonErrorHandler); + latch.await(10, TimeUnit.SECONDS); + + binderBindUnbindLatency(); + consumerBinding.unbind(); + producerBinding.unbind(); + } + + + //See https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/870 for motivation for this test. + @Test + @SuppressWarnings("unchecked") + public void testAutoCommitOnErrorWhenManualAcknowledgement() throws Exception { + Binder binder = getBinder(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + BindingProperties producerBindingProperties = createProducerBindingProperties( + producerProperties); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + producerBindingProperties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setMaxAttempts(3); + consumerProperties.setBackOffInitialInterval(100); + consumerProperties.setBackOffMaxInterval(150); + //When auto commit is disabled, then the record is committed after publishing to DLQ using the manual acknowledgement. + // (if DLQ is enabled, which is, in this case). + consumerProperties.getExtension().setAutoCommitOffset(false); + consumerProperties.getExtension().setEnableDlq(true); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler(); + moduleInputChannel.subscribe(handler); + long uniqueBindingId = System.currentTimeMillis(); + Binding producerBinding = binder.bindProducer( + "retryTest." + uniqueBindingId + ".0", moduleOutputChannel, + producerProperties); + Binding consumerBinding = binder.bindConsumer( + "retryTest." + uniqueBindingId + ".0", "testGroup", moduleInputChannel, + consumerProperties); + ExtendedConsumerProperties dlqConsumerProperties = createConsumerProperties(); + dlqConsumerProperties.setMaxAttempts(1); + QueueChannel dlqChannel = new QueueChannel(); + Binding dlqConsumerBinding = binder.bindConsumer( + "error.retryTest." + uniqueBindingId + ".0.testGroup", null, dlqChannel, + dlqConsumerProperties); + + String testMessagePayload = "test." + UUID.randomUUID().toString(); + Message testMessage = MessageBuilder + .withPayload(testMessagePayload.getBytes()).build(); + moduleOutputChannel.send(testMessage); + + Message dlqMessage = receive(dlqChannel, 3); + assertThat(dlqMessage).isNotNull(); + assertThat(dlqMessage.getPayload()).isEqualTo(testMessagePayload.getBytes()); + + // first attempt fails + assertThat(handler.getReceivedMessages().entrySet()).hasSize(1); + Message handledMessage = handler.getReceivedMessages().entrySet().iterator() + .next().getValue(); + assertThat(handledMessage).isNotNull(); + assertThat( + new String((byte[]) handledMessage.getPayload(), StandardCharsets.UTF_8)) + .isEqualTo(testMessagePayload); + assertThat(handler.getInvocationCount()) + .isEqualTo(consumerProperties.getMaxAttempts()); + binderBindUnbindLatency(); + dlqConsumerBinding.unbind(); + consumerBinding.unbind(); + + // on the second attempt the message is not redelivered because the DLQ is set and the record in error is already committed. + QueueChannel successfulInputChannel = new QueueChannel(); + consumerBinding = binder.bindConsumer("retryTest." + uniqueBindingId + ".0", + "testGroup", successfulInputChannel, consumerProperties); + String testMessage2Payload = "test1." + UUID.randomUUID().toString(); + Message testMessage2 = MessageBuilder + .withPayload(testMessage2Payload.getBytes()).build(); + moduleOutputChannel.send(testMessage2); + + Message receivedMessage = receive(successfulInputChannel); + assertThat(receivedMessage.getPayload()) + .isEqualTo(testMessage2Payload.getBytes()); + + binderBindUnbindLatency(); + consumerBinding.unbind(); + producerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testConfigurableDlqName() throws Exception { + Binder binder = getBinder(); + + ExtendedProducerProperties producerProperties = createProducerProperties(); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setMaxAttempts(3); + consumerProperties.setBackOffInitialInterval(100); + consumerProperties.setBackOffMaxInterval(150); + consumerProperties.getExtension().setEnableDlq(true); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + String dlqName = "dlqTest"; + consumerProperties.getExtension().setDlqName(dlqName); + BindingProperties producerBindingProperties = createProducerBindingProperties( + producerProperties); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + producerBindingProperties); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler(); + moduleInputChannel.subscribe(handler); + + long uniqueBindingId = System.currentTimeMillis(); + Binding producerBinding = binder.bindProducer( + "retryTest." + uniqueBindingId + ".0", moduleOutputChannel, + producerProperties); + Binding consumerBinding = binder.bindConsumer( + "retryTest." + uniqueBindingId + ".0", "testGroup", moduleInputChannel, + consumerProperties); + ExtendedConsumerProperties dlqConsumerProperties = createConsumerProperties(); + dlqConsumerProperties.setMaxAttempts(1); + QueueChannel dlqChannel = new QueueChannel(); + Binding dlqConsumerBinding = binder.bindConsumer(dlqName, null, + dlqChannel, dlqConsumerProperties); + + String testMessagePayload = "test." + UUID.randomUUID().toString(); + Message testMessage = MessageBuilder + .withPayload(testMessagePayload.getBytes()).build(); + moduleOutputChannel.send(testMessage); + + Message dlqMessage = receive(dlqChannel, 3); + assertThat(dlqMessage).isNotNull(); + assertThat(dlqMessage.getPayload()).isEqualTo(testMessagePayload.getBytes()); + + // first attempt fails + assertThat(handler.getReceivedMessages().entrySet()).hasSize(1); + Message handledMessage = handler.getReceivedMessages().entrySet().iterator() + .next().getValue(); + assertThat(handledMessage).isNotNull(); + assertThat( + new String((byte[]) handledMessage.getPayload(), StandardCharsets.UTF_8)) + .isEqualTo(testMessagePayload); + assertThat(handler.getInvocationCount()) + .isEqualTo(consumerProperties.getMaxAttempts()); + binderBindUnbindLatency(); + dlqConsumerBinding.unbind(); + consumerBinding.unbind(); + + // on the second attempt the message is not redelivered because the DLQ is set + QueueChannel successfulInputChannel = new QueueChannel(); + consumerBinding = binder.bindConsumer("retryTest." + uniqueBindingId + ".0", + "testGroup", successfulInputChannel, consumerProperties); + String testMessage2Payload = "test." + UUID.randomUUID().toString(); + Message testMessage2 = MessageBuilder + .withPayload(testMessage2Payload.getBytes()).build(); + moduleOutputChannel.send(testMessage2); + + Message receivedMessage = receive(successfulInputChannel); + assertThat(receivedMessage.getPayload()) + .isEqualTo(testMessage2Payload.getBytes()); + + binderBindUnbindLatency(); + consumerBinding.unbind(); + producerBinding.unbind(); + } + + @Test + public void testValidateKafkaTopicName() { + try { + KafkaTopicUtils.validateTopicName("foo:bar"); + fail("Expecting IllegalArgumentException"); + } + catch (Exception e) { + // TODO: handle exception + } + } + + @Test + @SuppressWarnings("unchecked") + // TODO: This test needs to be rethought - sending byte[] without explicit content + // type + // - yet being converted by the json converter + public void testCompression() throws Exception { + final KafkaProducerProperties.CompressionType[] codecs = new KafkaProducerProperties.CompressionType[] { + KafkaProducerProperties.CompressionType.none, + KafkaProducerProperties.CompressionType.gzip, + KafkaProducerProperties.CompressionType.snappy, + KafkaProducerProperties.CompressionType.zstd}; + byte[] testPayload = new byte[2048]; + Arrays.fill(testPayload, (byte) 65); + Binder binder = getBinder(); + for (KafkaProducerProperties.CompressionType codec : codecs) { + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.getExtension().setCompressionType( + KafkaProducerProperties.CompressionType.valueOf(codec.toString())); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + Binding producerBinding = binder.bindProducer( + "testCompression", moduleOutputChannel, producerProperties); + Binding consumerBinding = binder.bindConsumer( + "testCompression", "test", moduleInputChannel, consumerProperties); + Message message = org.springframework.integration.support.MessageBuilder + .withPayload(testPayload).build(); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + CountDownLatch latch = new CountDownLatch(1); + + AtomicReference> inboundMessageRef = new AtomicReference<>(); + moduleInputChannel.subscribe(message1 -> { + try { + inboundMessageRef.set((Message) message1); + } + finally { + latch.countDown(); + } + }); + Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message"); + + assertThat(inboundMessageRef.get()).isNotNull(); + assertThat(inboundMessageRef.get().getPayload()).containsExactly(testPayload); + producerBinding.unbind(); + consumerBinding.unbind(); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testEarliest() throws Exception { + Binding producerBinding = null; + Binding consumerBinding = null; + + try { + Binder binder = getBinder(); + BindingProperties producerBindingProperties = createProducerBindingProperties( + createProducerProperties()); + DirectChannel output = createBindableChannel("output", + producerBindingProperties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + consumerProperties.getExtension() + .setStartOffset(KafkaConsumerProperties.StartOffset.earliest); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + + DirectChannel input1 = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + String testTopicName = UUID.randomUUID().toString(); + producerBinding = binder.bindProducer(testTopicName, output, + producerBindingProperties.getProducer()); + String testPayload1 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload1.getBytes())); + + consumerBinding = binder.bindConsumer(testTopicName, "startOffsets", input1, + consumerProperties); + CountDownLatch latch = new CountDownLatch(1); + AtomicReference> inboundMessageRef1 = new AtomicReference<>(); + MessageHandler messageHandler = message1 -> { + try { + inboundMessageRef1.set((Message) message1); + } + finally { + latch.countDown(); + } + }; + input1.subscribe(messageHandler); + Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message"); + assertThat(inboundMessageRef1.get()).isNotNull(); + String testPayload2 = "foo-" + UUID.randomUUID().toString(); + input1.unsubscribe(messageHandler); + output.send(new GenericMessage<>(testPayload2.getBytes())); + + CountDownLatch latch1 = new CountDownLatch(1); + AtomicReference> inboundMessageRef2 = new AtomicReference<>(); + input1.subscribe(message1 -> { + try { + inboundMessageRef2.set((Message) message1); + } + finally { + latch1.countDown(); + } + }); + Assert.isTrue(latch1.await(5, TimeUnit.SECONDS), "Failed to receive message"); + + assertThat(inboundMessageRef2.get()).isNotNull(); + assertThat(new String(inboundMessageRef2.get().getPayload(), + StandardCharsets.UTF_8)).isEqualTo(testPayload2); + Thread.sleep(2000); + producerBinding.unbind(); + consumerBinding.unbind(); + } + finally { + if (consumerBinding != null) { + consumerBinding.unbind(); + } + if (producerBinding != null) { + producerBinding.unbind(); + } + } + } + + @Test + @Override + @SuppressWarnings("unchecked") + public void testSendAndReceiveMultipleTopics(TestInfo testInfo) throws Exception { + Binder binder = getBinder(); + + DirectChannel moduleOutputChannel1 = createBindableChannel("output1", + createProducerBindingProperties(createProducerProperties())); + DirectChannel moduleOutputChannel2 = createBindableChannel("output2", + createProducerBindingProperties(createProducerProperties())); + + QueueChannel moduleInputChannel = new QueueChannel(); + + ExtendedProducerProperties producer1Props = createProducerProperties(); + producer1Props.getExtension().setUseTopicHeader(true); + + Binding producerBinding1 = binder.bindProducer("foo.x", + moduleOutputChannel1, producer1Props); + Binding producerBinding2 = binder.bindProducer("foo.y", + moduleOutputChannel2, createProducerProperties()); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + Binding consumerBinding1 = binder.bindConsumer("foo.x", "test1", + moduleInputChannel, consumerProperties); + Binding consumerBinding2 = binder.bindConsumer("foo.y", "test2", + moduleInputChannel, consumerProperties); + + String testPayload1 = "foo1"; + Message message1 = org.springframework.integration.support.MessageBuilder + .withPayload(testPayload1.getBytes()).build(); + String testPayload2 = "foo2"; + Message message2 = org.springframework.integration.support.MessageBuilder + .withPayload(testPayload2.getBytes()).build(); + String testPayload3 = "foo3"; + Message message3 = org.springframework.integration.support.MessageBuilder + .withPayload(testPayload3.getBytes()) + .setHeader(KafkaHeaders.TOPIC, "foo.y") + .build(); + + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel1.send(message1); + moduleOutputChannel2.send(message2); + moduleOutputChannel1.send(message3); + + Message[] messages = new Message[3]; + messages[0] = receive(moduleInputChannel); + messages[1] = receive(moduleInputChannel); + messages[2] = receive(moduleInputChannel); + + assertThat(messages[0]).isNotNull(); + assertThat(messages[1]).isNotNull(); + assertThat(messages[1]).isNotNull(); + assertThat(messages).extracting("payload").containsExactlyInAnyOrder( + testPayload1.getBytes(), testPayload2.getBytes(), testPayload3.getBytes()); + Arrays.asList(messages).forEach(message -> { + if (new String((byte[]) message.getPayload()).equals("foo1")) { + assertThat(message.getHeaders().get(KafkaHeaders.RECEIVED_TOPIC)).isEqualTo("foo.x"); + } + else { + assertThat(message.getHeaders().get(KafkaHeaders.RECEIVED_TOPIC)).isEqualTo("foo.y"); + } + }); + + producerBinding1.unbind(); + producerBinding2.unbind(); + + consumerBinding1.unbind(); + consumerBinding2.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testManualAckSucceedsWhenAutoCommitOffsetIsTurnedOff() throws Exception { + Binder binder = getBinder(); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(createProducerProperties())); + QueueChannel moduleInputChannel = new QueueChannel(); + + Binding producerBinding = binder.bindProducer( + "testManualAckSucceedsWhenAutoCommitOffsetIsTurnedOff", + moduleOutputChannel, createProducerProperties()); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.getExtension().setAckMode(ContainerProperties.AckMode.MANUAL); + + Binding consumerBinding = binder.bindConsumer( + "testManualAckSucceedsWhenAutoCommitOffsetIsTurnedOff", "test", + moduleInputChannel, consumerProperties); + + String testPayload1 = "foo" + UUID.randomUUID().toString(); + Message message1 = org.springframework.integration.support.MessageBuilder + .withPayload(testPayload1.getBytes()).build(); + + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message1); + + Message receivedMessage = receive(moduleInputChannel); + assertThat(receivedMessage).isNotNull(); + assertThat(receivedMessage.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT)) + .isNotNull(); + Acknowledgment acknowledgment = receivedMessage.getHeaders() + .get(KafkaHeaders.ACKNOWLEDGMENT, Acknowledgment.class); + try { + acknowledgment.acknowledge(); + } + catch (Exception e) { + fail("Acknowledge must not throw an exception"); + } + finally { + producerBinding.unbind(); + consumerBinding.unbind(); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testManualAckIsNotPossibleWhenAutoCommitOffsetIsEnabledOnTheBinder() + throws Exception { + Binder binder = getBinder(); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(createProducerProperties())); + QueueChannel moduleInputChannel = new QueueChannel(); + + Binding producerBinding = binder.bindProducer( + "testManualAckIsNotPossibleWhenAutoCommitOffsetIsEnabledOnTheBinder", + moduleOutputChannel, createProducerProperties()); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + + Binding consumerBinding = binder.bindConsumer( + "testManualAckIsNotPossibleWhenAutoCommitOffsetIsEnabledOnTheBinder", + "test", moduleInputChannel, consumerProperties); + + AbstractMessageListenerContainer container = TestUtils.getPropertyValue( + consumerBinding, "lifecycle.messageListenerContainer", + AbstractMessageListenerContainer.class); + assertThat(container.getContainerProperties().getAckMode()) + .isEqualTo(ContainerProperties.AckMode.BATCH); + + String testPayload1 = "foo" + UUID.randomUUID().toString(); + Message message1 = org.springframework.integration.support.MessageBuilder + .withPayload(testPayload1.getBytes()).build(); + + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message1); + + Message receivedMessage = receive(moduleInputChannel); + assertThat(receivedMessage).isNotNull(); + assertThat(receivedMessage.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT)) + .isNull(); + + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + @Override + @SuppressWarnings("unchecked") + public void testTwoRequiredGroups(TestInfo testInfo) throws Exception { + Binder binder = getBinder(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + + DirectChannel output = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + + String testDestination = "testDestination" + + UUID.randomUUID().toString().replace("-", ""); + + producerProperties.setRequiredGroups("test1", "test2"); + Binding producerBinding = binder.bindProducer(testDestination, + output, producerProperties); + + String testPayload = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload.getBytes())); + + QueueChannel inbound1 = new QueueChannel(); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + consumerProperties.getExtension().setAckMode(ContainerProperties.AckMode.RECORD); + Binding consumerBinding1 = binder.bindConsumer(testDestination, + "test1", inbound1, consumerProperties); + QueueChannel inbound2 = new QueueChannel(); + Binding consumerBinding2 = binder.bindConsumer(testDestination, + "test2", inbound2, consumerProperties); + + AbstractMessageListenerContainer container = TestUtils.getPropertyValue( + consumerBinding2, "lifecycle.messageListenerContainer", + AbstractMessageListenerContainer.class); + assertThat(container.getContainerProperties().getAckMode()) + .isEqualTo(ContainerProperties.AckMode.RECORD); + + Message receivedMessage1 = receive(inbound1); + assertThat(receivedMessage1).isNotNull(); + assertThat(new String((byte[]) receivedMessage1.getPayload(), + StandardCharsets.UTF_8)).isEqualTo(testPayload); + Message receivedMessage2 = receive(inbound2); + assertThat(receivedMessage2).isNotNull(); + assertThat(new String((byte[]) receivedMessage2.getPayload(), + StandardCharsets.UTF_8)).isEqualTo(testPayload); + + consumerBinding1.unbind(); + consumerBinding2.unbind(); + producerBinding.unbind(); + } + + @Test + @Override + @SuppressWarnings("unchecked") + public void testPartitionedModuleSpEL(TestInfo testInfo) throws Exception { + Binder binder = getBinder(); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setConcurrency(2); + consumerProperties.setInstanceIndex(0); + consumerProperties.setInstanceCount(3); + consumerProperties.setPartitioned(true); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + QueueChannel input0 = new QueueChannel(); + input0.setBeanName("test.input0S"); + Binding input0Binding = binder.bindConsumer("part.0", "test", + input0, consumerProperties); + consumerProperties.setInstanceIndex(1); + QueueChannel input1 = new QueueChannel(); + input1.setBeanName("test.input1S"); + Binding input1Binding = binder.bindConsumer("part.0", "test", + input1, consumerProperties); + consumerProperties.setInstanceIndex(2); + QueueChannel input2 = new QueueChannel(); + input2.setBeanName("test.input2S"); + Binding input2Binding = binder.bindConsumer("part.0", "test", + input2, consumerProperties); + + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setPartitionKeyExpression( + spelExpressionParser.parseExpression("payload")); + producerProperties.setPartitionSelectorExpression( + spelExpressionParser.parseExpression("hashCode()")); + producerProperties.setPartitionCount(3); + invokeCreateTopic("output", 6, 1); + + DirectChannel output = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + output.setBeanName("test.output"); + Binding outputBinding = binder.bindProducer("part.0", output, + producerProperties); + try { + Object endpoint = extractEndpoint(outputBinding); + assertThat(getEndpointRouting(endpoint)) + .contains(getExpectedRoutingBaseDestination("part.0", "test") + + "-' + headers['partition']"); + } + catch (UnsupportedOperationException ignored) { + } + List interceptors = output.getInterceptors(); + AtomicInteger count = new AtomicInteger(); + interceptors.forEach(interceptor -> { + if (interceptor instanceof PartitioningInterceptor) { + count.set(TestUtils.getPropertyValue(interceptor, + "partitionHandler.partitionCount", Integer.class)); + } + }); + assertThat(count.get()).isEqualTo(6); + Message message2 = org.springframework.integration.support.MessageBuilder + .withPayload(2) + .setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID, "foo") + .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, 42) + .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, 43).build(); + output.send(message2); + output.send(new GenericMessage<>(1)); + output.send(new GenericMessage<>(0)); + + Message receive0 = receive(input0); + assertThat(receive0).isNotNull(); + Message receive1 = receive(input1); + assertThat(receive1).isNotNull(); + Message receive2 = receive(input2); + assertThat(receive2).isNotNull(); + + Condition> correlationHeadersForPayload2 = new Condition>() { + + @Override + public boolean matches(Message value) { + IntegrationMessageHeaderAccessor accessor = new IntegrationMessageHeaderAccessor( + value); + return "foo".equals(accessor.getCorrelationId()) + && 42 == accessor.getSequenceNumber() + && 43 == accessor.getSequenceSize(); + } + }; + + ObjectMapper om = new ObjectMapper(); + + if (usesExplicitRouting()) { + assertThat(om.readValue((byte[]) receive0.getPayload(), Integer.class)) + .isEqualTo(0); + assertThat(om.readValue((byte[]) receive1.getPayload(), Integer.class)) + .isEqualTo(1); + assertThat(om.readValue((byte[]) receive2.getPayload(), Integer.class)) + .isEqualTo(2); + assertThat(receive2).has(correlationHeadersForPayload2); + } + else { + List> receivedMessages = Arrays.asList(receive0, receive1, + receive2); + assertThat(receivedMessages).extracting("payload").containsExactlyInAnyOrder( + new byte[] { 48 }, new byte[] { 49 }, new byte[] { 50 }); + Condition> payloadIs2 = new Condition>() { + + @Override + public boolean matches(Message value) { + try { + return om.readValue((byte[]) value.getPayload(), Integer.class) + .equals(2); + } + catch (IOException e) { + // + } + return false; + } + }; + assertThat(receivedMessages).filteredOn(payloadIs2).areExactly(1, + correlationHeadersForPayload2); + + } + input0Binding.unbind(); + input1Binding.unbind(); + input2Binding.unbind(); + outputBinding.unbind(); + } + + @Test +// @Override + @SuppressWarnings({ "unchecked", "rawtypes" }) + public void testPartitionedModuleJava() throws Exception { + Binder binder = getBinder(); + + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setConcurrency(2); + consumerProperties.setInstanceCount(4); + consumerProperties.setInstanceIndex(0); + consumerProperties.setPartitioned(true); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + QueueChannel input0 = new QueueChannel(); + input0.setBeanName("test.input0J"); + Binding input0Binding = binder.bindConsumer("partJ.0", "test", + input0, consumerProperties); + consumerProperties.setInstanceIndex(1); + QueueChannel input1 = new QueueChannel(); + input1.setBeanName("test.input1J"); + Binding input1Binding = binder.bindConsumer("partJ.0", "test", + input1, consumerProperties); + consumerProperties.setInstanceIndex(2); + QueueChannel input2 = new QueueChannel(); + input2.setBeanName("test.input2J"); + Binding input2Binding = binder.bindConsumer("partJ.0", "test", + input2, consumerProperties); + consumerProperties.setInstanceIndex(3); + QueueChannel input3 = new QueueChannel(); + input3.setBeanName("test.input3J"); + Binding input3Binding = binder.bindConsumer("partJ.0", "test", + input3, consumerProperties); + + ExtendedProducerProperties producerProperties = createProducerProperties(); + + this.applicationContext.registerBean("pkExtractor", + PartitionTestSupport.class, () -> new PartitionTestSupport()); + this.applicationContext.registerBean("pkSelector", + PartitionTestSupport.class, () -> new PartitionTestSupport()); + producerProperties.setPartitionKeyExtractorName("pkExtractor"); + producerProperties.setPartitionSelectorName("pkSelector"); + producerProperties.setPartitionCount(3); // overridden to 8 on the actual topic + DirectChannel output = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + output.setBeanName("test.output"); + Binding outputBinding = binder.bindProducer("partJ.0", output, + producerProperties); + if (usesExplicitRouting()) { + Object endpoint = extractEndpoint(outputBinding); + assertThat(getEndpointRouting(endpoint)) + .contains(getExpectedRoutingBaseDestination("partJ.0", "test") + + "-' + headers['partition']"); + } + + output.send(new GenericMessage<>(2)); + output.send(new GenericMessage<>(1)); + output.send(new GenericMessage<>(0)); + output.send(new GenericMessage<>(3)); + + Message receive0 = receive(input0); + assertThat(receive0).isNotNull(); + Message receive1 = receive(input1); + assertThat(receive1).isNotNull(); + Message receive2 = receive(input2); + assertThat(receive2).isNotNull(); + Message receive3 = receive(input3); + assertThat(receive3).isNotNull(); + ObjectMapper om = new ObjectMapper(); + + assertThat(om.readValue((byte[]) receive0.getPayload(), Integer.class)) + .isEqualTo(0); + assertThat(om.readValue((byte[]) receive1.getPayload(), Integer.class)) + .isEqualTo(1); + assertThat(om.readValue((byte[]) receive2.getPayload(), Integer.class)) + .isEqualTo(2); + assertThat(om.readValue((byte[]) receive3.getPayload(), Integer.class)) + .isEqualTo(3); + + input0Binding.unbind(); + input1Binding.unbind(); + input2Binding.unbind(); + input3Binding.unbind(); + outputBinding.unbind(); + } + + @Test + @Override + @SuppressWarnings("unchecked") + public void testAnonymousGroup(TestInfo testInfo) throws Exception { + Binder binder = getBinder(); + BindingProperties producerBindingProperties = createProducerBindingProperties( + createProducerProperties()); + DirectChannel output = createBindableChannel("output", producerBindingProperties); + Binding producerBinding = binder.bindProducer("defaultGroup.0", + output, producerBindingProperties.getProducer()); + + QueueChannel input1 = new QueueChannel(); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + Binding binding1 = binder.bindConsumer("defaultGroup.0", null, + input1, consumerProperties); + + QueueChannel input2 = new QueueChannel(); + Binding binding2 = binder.bindConsumer("defaultGroup.0", null, + input2, consumerProperties); + // Since we don't provide any topic info, let Kafka bind the consumer successfully + Thread.sleep(1000); + String testPayload1 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload1.getBytes())); + + Message receivedMessage1 = (Message) receive(input1); + assertThat(receivedMessage1).isNotNull(); + assertThat(new String(receivedMessage1.getPayload(), StandardCharsets.UTF_8)) + .isEqualTo(testPayload1); + + Message receivedMessage2 = (Message) receive(input2); + assertThat(receivedMessage2).isNotNull(); + assertThat(new String(receivedMessage2.getPayload(), StandardCharsets.UTF_8)) + .isEqualTo(testPayload1); + + binding2.unbind(); + + String testPayload2 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload2.getBytes())); + + binding2 = binder.bindConsumer("defaultGroup.0", null, input2, + consumerProperties); + // Since we don't provide any topic info, let Kafka bind the consumer successfully + Thread.sleep(1000); + String testPayload3 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload3.getBytes())); + + receivedMessage1 = (Message) receive(input1); + assertThat(receivedMessage1).isNotNull(); + assertThat(new String(receivedMessage1.getPayload(), StandardCharsets.UTF_8)) + .isEqualTo(testPayload2); + receivedMessage1 = (Message) receive(input1); + assertThat(receivedMessage1).isNotNull(); + assertThat(new String(receivedMessage1.getPayload(), StandardCharsets.UTF_8)) + .isNotNull(); + + receivedMessage2 = (Message) receive(input2); + assertThat(receivedMessage2).isNotNull(); + assertThat(new String(receivedMessage2.getPayload(), StandardCharsets.UTF_8)) + .isEqualTo(testPayload3); + + Map topicsInUse = ((KafkaTestBinder) binder) + .getCoreBinder().getTopicsInUse(); + assertThat(topicsInUse.keySet()).contains("defaultGroup.0"); + TopicInformation topic = topicsInUse.get("defaultGroup.0"); + assertThat(topic.isConsumerTopic()).isTrue(); + assertThat(topic.getConsumerGroup()).startsWith("anonymous"); + + producerBinding.unbind(); + binding1.unbind(); + binding2.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testPartitionedModuleJavaWithRawMode() throws Exception { + Binder binder = getBinder(); + ExtendedProducerProperties properties = createProducerProperties(); + properties.setHeaderMode(HeaderMode.none); + this.applicationContext.registerBean("pkExtractor", + RawKafkaPartitionTestSupport.class, () -> new RawKafkaPartitionTestSupport()); + this.applicationContext.registerBean("pkSelector", + RawKafkaPartitionTestSupport.class, () -> new RawKafkaPartitionTestSupport()); + properties.setPartitionKeyExtractorName("pkExtractor"); + properties.setPartitionSelectorName("pkSelector"); + properties.setPartitionCount(6); + + DirectChannel output = createBindableChannel("output", + createProducerBindingProperties(properties)); + output.setBeanName("test.output"); + Binding outputBinding = binder.bindProducer("partJ.raw.0", output, + properties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setConcurrency(2); + consumerProperties.setInstanceCount(3); + consumerProperties.setInstanceIndex(0); + consumerProperties.setPartitioned(true); + consumerProperties.setHeaderMode(HeaderMode.none); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + QueueChannel input0 = new QueueChannel(); + input0.setBeanName("test.input0J"); + Binding input0Binding = binder.bindConsumer("partJ.raw.0", "test", + input0, consumerProperties); + consumerProperties.setInstanceIndex(1); + QueueChannel input1 = new QueueChannel(); + input1.setBeanName("test.input1J"); + Binding input1Binding = binder.bindConsumer("partJ.raw.0", "test", + input1, consumerProperties); + consumerProperties.setInstanceIndex(2); + QueueChannel input2 = new QueueChannel(); + input2.setBeanName("test.input2J"); + Binding input2Binding = binder.bindConsumer("partJ.raw.0", "test", + input2, consumerProperties); + + output.send(new GenericMessage<>(new byte[] { (byte) 0 })); + output.send(new GenericMessage<>(new byte[] { (byte) 1 })); + output.send(new GenericMessage<>(new byte[] { (byte) 2 })); + + Message receive0 = receive(input0); + assertThat(receive0).isNotNull(); + Message receive1 = receive(input1); + assertThat(receive1).isNotNull(); + Message receive2 = receive(input2); + assertThat(receive2).isNotNull(); + + assertThat(Arrays.asList(((byte[]) receive0.getPayload())[0], + ((byte[]) receive1.getPayload())[0], ((byte[]) receive2.getPayload())[0])) + .containsExactlyInAnyOrder((byte) 0, (byte) 1, (byte) 2); + + input0Binding.unbind(); + input1Binding.unbind(); + input2Binding.unbind(); + outputBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testPartitionedModuleSpELWithRawMode() throws Exception { + Binder binder = getBinder(); + ExtendedProducerProperties properties = createProducerProperties(); + properties.setPartitionKeyExpression( + spelExpressionParser.parseExpression("payload[0]")); + properties.setPartitionSelectorExpression( + spelExpressionParser.parseExpression("hashCode()")); + properties.setPartitionCount(6); + properties.setHeaderMode(HeaderMode.none); + + DirectChannel output = createBindableChannel("output", + createProducerBindingProperties(properties)); + output.setBeanName("test.output"); + Binding outputBinding = binder.bindProducer("part.raw.0", output, + properties); + try { + Object endpoint = extractEndpoint(outputBinding); + assertThat(getEndpointRouting(endpoint)) + .contains(getExpectedRoutingBaseDestination("part.raw.0", "test") + + "-' + headers['partition']"); + } + catch (UnsupportedOperationException ignored) { + } + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setConcurrency(2); + consumerProperties.setInstanceIndex(0); + consumerProperties.setInstanceCount(3); + consumerProperties.setPartitioned(true); + consumerProperties.setHeaderMode(HeaderMode.none); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + QueueChannel input0 = new QueueChannel(); + input0.setBeanName("test.input0S"); + Binding input0Binding = binder.bindConsumer("part.raw.0", "test", + input0, consumerProperties); + consumerProperties.setInstanceIndex(1); + QueueChannel input1 = new QueueChannel(); + input1.setBeanName("test.input1S"); + Binding input1Binding = binder.bindConsumer("part.raw.0", "test", + input1, consumerProperties); + consumerProperties.setInstanceIndex(2); + QueueChannel input2 = new QueueChannel(); + input2.setBeanName("test.input2S"); + Binding input2Binding = binder.bindConsumer("part.raw.0", "test", + input2, consumerProperties); + + Message message2 = org.springframework.integration.support.MessageBuilder + .withPayload(new byte[] { 2 }) + .setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID, + "kafkaBinderTestCommonsDelegate") + .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, 42) + .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, 43).build(); + output.send(message2); + output.send(new GenericMessage<>(new byte[] { 1 })); + output.send(new GenericMessage<>(new byte[] { 0 })); + Message receive0 = receive(input0); + assertThat(receive0).isNotNull(); + Message receive1 = receive(input1); + assertThat(receive1).isNotNull(); + Message receive2 = receive(input2); + assertThat(receive2).isNotNull(); + assertThat(Arrays.asList(((byte[]) receive0.getPayload())[0], + ((byte[]) receive1.getPayload())[0], ((byte[]) receive2.getPayload())[0])) + .containsExactlyInAnyOrder((byte) 0, (byte) 1, (byte) 2); + input0Binding.unbind(); + input1Binding.unbind(); + input2Binding.unbind(); + outputBinding.unbind(); + } + + @Test + @SuppressWarnings({ "unchecked", "rawtypes" }) + public void testPartitionedNative() throws Exception { + Binder binder = getBinder(); + ExtendedProducerProperties properties = createProducerProperties(); + properties.setPartitionCount(6); + + DirectChannel output = createBindableChannel("output", + createProducerBindingProperties(properties)); + output.setBeanName("test.output"); + Binding outputBinding = binder.bindProducer("partNative.raw.0", + output, properties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + QueueChannel input0 = new QueueChannel(); + input0.setBeanName("test.inputNative"); + Binding inputBinding = binder.bindConsumer("partNative.raw.0", + "test", input0, consumerProperties); + + output.send(new GenericMessage<>("foo".getBytes(), + Collections.singletonMap(KafkaHeaders.PARTITION_ID, 5))); + + Message received = receive(input0); + assertThat(received).isNotNull(); + + assertThat(received.getPayload()).isEqualTo("foo".getBytes()); + assertThat(received.getHeaders().get(KafkaHeaders.RECEIVED_PARTITION_ID)) + .isEqualTo(5); + + inputBinding.unbind(); + outputBinding.unbind(); + } + + @Test + @SuppressWarnings({ "unchecked", "rawtypes" }) + public void testSendAndReceiveWithRawMode() throws Exception { + Binder binder = getBinder(); + + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setHeaderMode(HeaderMode.none); + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setHeaderMode(HeaderMode.none); + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + Binding producerBinding = binder.bindProducer("raw.0", + moduleOutputChannel, producerProperties); + + Binding consumerBinding = binder.bindConsumer("raw.0", "test", + moduleInputChannel, consumerProperties); + Message message = org.springframework.integration.support.MessageBuilder + .withPayload("testSendAndReceiveWithRawMode".getBytes()).build(); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference> inboundMessageRef = new AtomicReference<>(); + moduleInputChannel.subscribe(message1 -> { + try { + inboundMessageRef.set((Message) message1); + } + finally { + latch.countDown(); + } + }); + Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message"); + + assertThat(inboundMessageRef.get()).isNotNull(); + assertThat( + new String(inboundMessageRef.get().getPayload(), StandardCharsets.UTF_8)) + .isEqualTo("testSendAndReceiveWithRawMode"); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Test + public void testAllowNonTransactionalProducerSetting() throws Exception { + AbstractKafkaTestBinder binder = getBinder(); + DirectChannel moduleOutputChannel = createBindableChannel("output", + new BindingProperties()); + ExtendedProducerProperties producerProps = new ExtendedProducerProperties<>( + new KafkaProducerProperties()); + producerProps.getExtension().setAllowNonTransactional(true); + Binding producerBinding = binder.bindProducer("allwNonTrans.0", + moduleOutputChannel, producerProps); + + KafkaProducerMessageHandler endpoint = TestUtils.getPropertyValue(producerBinding, + "lifecycle", KafkaProducerMessageHandler.class); + + final KafkaTemplate kafkaTemplate = (KafkaTemplate) new DirectFieldAccessor(endpoint).getPropertyValue("kafkaTemplate"); + + assertThat(kafkaTemplate.isAllowNonTransactional()).isTrue(); + + producerBinding.unbind(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Test + public void testProducerErrorChannel() throws Exception { + AbstractKafkaTestBinder binder = getBinder(); + DirectChannel moduleOutputChannel = createBindableChannel("output", + new BindingProperties()); + ExtendedProducerProperties producerProps = new ExtendedProducerProperties<>( + new KafkaProducerProperties()); + producerProps.setHeaderMode(HeaderMode.none); + producerProps.setErrorChannelEnabled(true); + Binding producerBinding = binder.bindProducer("ec.0", + moduleOutputChannel, producerProps); + final Message message = MessageBuilder.withPayload("bad") + .setHeader(MessageHeaders.CONTENT_TYPE, "application/json").build(); + SubscribableChannel ec = binder.getApplicationContext().getBean("ec.0.errors", + SubscribableChannel.class); + final AtomicReference> errorMessage = new AtomicReference<>(); + final CountDownLatch latch = new CountDownLatch(2); + ec.subscribe(message1 -> { + errorMessage.set(message1); + latch.countDown(); + }); + SubscribableChannel globalEc = binder.getApplicationContext().getBean( + IntegrationContextUtils.ERROR_CHANNEL_BEAN_NAME, + SubscribableChannel.class); + globalEc.subscribe(message12 -> latch.countDown()); + KafkaProducerMessageHandler endpoint = TestUtils.getPropertyValue(producerBinding, + "lifecycle", KafkaProducerMessageHandler.class); + final RuntimeException fooException = new RuntimeException("foo"); + final AtomicReference sent = new AtomicReference<>(); + new DirectFieldAccessor(endpoint).setPropertyValue("kafkaTemplate", + new KafkaTemplate(mock(ProducerFactory.class)) { + + @Override // SIK < 2.3 + public ListenableFuture send(String topic, + Object payload) { + sent.set(payload); + SettableListenableFuture future = new SettableListenableFuture<>(); + future.setException(fooException); + return future; + } + + @Override // SIK 2.3+ + public ListenableFuture send(ProducerRecord record) { + sent.set(record.value()); + SettableListenableFuture future = new SettableListenableFuture<>(); + future.setException(fooException); + return future; + } + + }); + + moduleOutputChannel.send(message); + assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(errorMessage.get()).isInstanceOf(ErrorMessage.class); + assertThat(errorMessage.get().getPayload()) + .isInstanceOf(KafkaSendFailureException.class); + KafkaSendFailureException exception = (KafkaSendFailureException) errorMessage + .get().getPayload(); + assertThat(exception.getCause()).isSameAs(fooException); + assertThat(new String((byte[]) exception.getFailedMessage().getPayload(), + StandardCharsets.UTF_8)).isEqualTo(message.getPayload()); + assertThat(exception.getRecord().value()).isSameAs(sent.get()); + producerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testAutoCreateTopicsEnabledSucceeds() throws Exception { + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + configurationProperties.setAutoCreateTopics(true); + Binder binder = getBinder(configurationProperties); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + String testTopicName = "nonexisting" + System.currentTimeMillis(); + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + Binding binding = binder.bindConsumer(testTopicName, "test", + moduleInputChannel, consumerProperties); + binding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testCustomPartitionCountOverridesDefaultIfLarger() throws Exception { + byte[] testPayload = new byte[2048]; + Arrays.fill(testPayload, (byte) 65); + KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties(); + binderConfiguration.setMinPartitionCount(10); + Binder binder = getBinder(binderConfiguration); + QueueChannel moduleInputChannel = new QueueChannel(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setPartitionCount(10); + producerProperties.setPartitionKeyExpression(new LiteralExpression("foo")); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + long uniqueBindingId = System.currentTimeMillis(); + Binding producerBinding = binder.bindProducer( + "foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties); + Binding consumerBinding = binder.bindConsumer( + "foo" + uniqueBindingId + ".0", null, moduleInputChannel, + consumerProperties); + Message message = org.springframework.integration.support.MessageBuilder + .withPayload(testPayload).build(); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = receive(moduleInputChannel); + assertThat(inbound).isNotNull(); + assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload); + + assertThat(partitionSize("foo" + uniqueBindingId + ".0")).isEqualTo(10); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testCustomPartitionCountDoesNotOverridePartitioningIfSmaller() + throws Exception { + byte[] testPayload = new byte[2048]; + Arrays.fill(testPayload, (byte) 65); + KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties(); + binderConfiguration.setMinPartitionCount(6); + Binder binder = getBinder(binderConfiguration); + QueueChannel moduleInputChannel = new QueueChannel(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setPartitionCount(5); + producerProperties.setPartitionKeyExpression( + spelExpressionParser.parseExpression("payload")); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + long uniqueBindingId = System.currentTimeMillis(); + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + Binding producerBinding = binder.bindProducer( + "foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties); + Binding consumerBinding = binder.bindConsumer( + "foo" + uniqueBindingId + ".0", null, moduleInputChannel, + consumerProperties); + Thread.sleep(1000); + Message message = org.springframework.integration.support.MessageBuilder + .withPayload(testPayload).build(); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = receive(moduleInputChannel); + assertThat(inbound).isNotNull(); + assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload); + + assertThat(partitionSize("foo" + uniqueBindingId + ".0")).isEqualTo(6); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testDynamicKeyExpression() throws Exception { + Binder binder = getBinder(createConfigurationProperties()); + QueueChannel moduleInputChannel = new QueueChannel(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.getExtension().getConfiguration().put("key.serializer", + StringSerializer.class.getName()); + producerProperties.getExtension().setMessageKeyExpression( + spelExpressionParser.parseExpression("headers.key")); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + String uniqueBindingId = UUID.randomUUID().toString(); + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + Binding producerBinding = binder.bindProducer( + "foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties); + Binding consumerBinding = binder.bindConsumer( + "foo" + uniqueBindingId + ".0", null, moduleInputChannel, + consumerProperties); + Thread.sleep(1000); + Message message = MessageBuilder.withPayload("somePayload") + .setHeader("key", "myDynamicKey").build(); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = receive(moduleInputChannel); + assertThat(inbound).isNotNull(); + String receivedKey = new String(inbound.getHeaders() + .get(KafkaHeaders.RECEIVED_MESSAGE_KEY, byte[].class)); + assertThat(receivedKey).isEqualTo("myDynamicKey"); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testCustomPartitionCountOverridesPartitioningIfLarger() throws Exception { + byte[] testPayload = new byte[2048]; + Arrays.fill(testPayload, (byte) 65); + KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties(); + binderConfiguration.setMinPartitionCount(4); + Binder binder = getBinder(binderConfiguration); + + QueueChannel moduleInputChannel = new QueueChannel(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setPartitionCount(5); + producerProperties.setPartitionKeyExpression( + spelExpressionParser.parseExpression("payload")); + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + long uniqueBindingId = System.currentTimeMillis(); + Binding producerBinding = binder.bindProducer( + "foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties); + Binding consumerBinding = binder.bindConsumer( + "foo" + uniqueBindingId + ".0", null, moduleInputChannel, + consumerProperties); + Message message = org.springframework.integration.support.MessageBuilder + .withPayload(testPayload).build(); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = receive(moduleInputChannel); + assertThat(inbound).isNotNull(); + assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload); + assertThat(partitionSize("foo" + uniqueBindingId + ".0")).isEqualTo(5); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testDefaultConsumerStartsAtEarliest() throws Exception { + Binder binder = getBinder(createConfigurationProperties()); + + BindingProperties producerBindingProperties = createProducerBindingProperties( + createProducerProperties()); + DirectChannel output = createBindableChannel("output", producerBindingProperties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + + DirectChannel input1 = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + String testTopicName = UUID.randomUUID().toString(); + Binding producerBinding = binder.bindProducer(testTopicName, + output, createProducerProperties()); + String testPayload1 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload1.getBytes())); + + Binding consumerBinding = binder.bindConsumer(testTopicName, + "startOffsets", input1, consumerProperties); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference> inboundMessageRef1 = new AtomicReference<>(); + MessageHandler messageHandler = message1 -> { + try { + inboundMessageRef1.set((Message) message1); + } + finally { + latch.countDown(); + } + }; + input1.subscribe(messageHandler); + Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message"); + assertThat(inboundMessageRef1.get()).isNotNull(); + assertThat( + new String(inboundMessageRef1.get().getPayload(), StandardCharsets.UTF_8)) + .isEqualTo(testPayload1); + + String testPayload2 = "foo-" + UUID.randomUUID().toString(); + input1.unsubscribe(messageHandler); + output.send(new GenericMessage<>(testPayload2.getBytes())); + + CountDownLatch latch1 = new CountDownLatch(1); + AtomicReference> inboundMessageRef2 = new AtomicReference<>(); + input1.subscribe(message1 -> { + try { + inboundMessageRef2.set((Message) message1); + } + finally { + latch1.countDown(); + } + }); + Assert.isTrue(latch1.await(5, TimeUnit.SECONDS), "Failed to receive message"); + + assertThat(inboundMessageRef2.get()).isNotNull(); + assertThat( + new String(inboundMessageRef2.get().getPayload(), StandardCharsets.UTF_8)) + .isEqualTo(testPayload2); + + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testResume() throws Exception { + Binding producerBinding = null; + Binding consumerBinding = null; + + try { + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + Binder binder = getBinder(configurationProperties); + + BindingProperties producerBindingProperties = createProducerBindingProperties( + createProducerProperties()); + DirectChannel output = createBindableChannel("output", + producerBindingProperties); + + DirectChannel input1 = createBindableChannel("input", + createConsumerBindingProperties(createConsumerProperties())); + + String testTopicName = UUID.randomUUID().toString(); + producerBinding = binder.bindProducer(testTopicName, output, + producerBindingProperties.getProducer()); + ExtendedConsumerProperties firstConsumerProperties = createConsumerProperties(); + consumerBinding = binder.bindConsumer(testTopicName, "startOffsets", input1, + firstConsumerProperties); + CountDownLatch latch = new CountDownLatch(1); + AtomicReference> inboundMessageRef1 = new AtomicReference<>(); + MessageHandler messageHandler = message1 -> { + try { + inboundMessageRef1.set((Message) message1); + } + finally { + latch.countDown(); + } + }; + input1.subscribe(messageHandler); + String testPayload1 = "foo1-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload1)); + Assert.isTrue(latch.await(15, TimeUnit.SECONDS), "Failed to receive message"); + + assertThat(inboundMessageRef1.get()).isNotNull(); + assertThat(inboundMessageRef1.get().getPayload()).isNotNull(); + input1.unsubscribe(messageHandler); + CountDownLatch latch1 = new CountDownLatch(1); + AtomicReference> inboundMessageRef2 = new AtomicReference<>(); + MessageHandler messageHandler1 = message1 -> { + try { + inboundMessageRef2.set((Message) message1); + } + finally { + latch1.countDown(); + } + }; + input1.subscribe(messageHandler1); + String testPayload2 = "foo2-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload2.getBytes())); + Assert.isTrue(latch1.await(15, TimeUnit.SECONDS), + "Failed to receive message"); + assertThat(inboundMessageRef2.get()).isNotNull(); + assertThat(inboundMessageRef2.get().getPayload()).isNotNull(); + consumerBinding.unbind(); + + Thread.sleep(2000); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerBinding = binder.bindConsumer(testTopicName, "startOffsets", input1, + consumerProperties); + input1.unsubscribe(messageHandler1); + CountDownLatch latch2 = new CountDownLatch(1); + AtomicReference> inboundMessageRef3 = new AtomicReference<>(); + MessageHandler messageHandler2 = message1 -> { + try { + inboundMessageRef3.set((Message) message1); + } + finally { + latch2.countDown(); + } + }; + input1.subscribe(messageHandler2); + String testPayload3 = "foo3-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload3.getBytes())); + Assert.isTrue(latch2.await(15, TimeUnit.SECONDS), + "Failed to receive message"); + assertThat(inboundMessageRef3.get()).isNotNull(); + assertThat(new String(inboundMessageRef3.get().getPayload(), + StandardCharsets.UTF_8)).isEqualTo(testPayload3); + } + finally { + if (consumerBinding != null) { + consumerBinding.unbind(); + } + if (producerBinding != null) { + producerBinding.unbind(); + } + } + } + + @Test + @SuppressWarnings("unchecked") + public void testSyncProducerMetadata() throws Exception { + Binder binder = getBinder(createConfigurationProperties()); + DirectChannel output = new DirectChannel(); + String testTopicName = UUID.randomUUID().toString(); + ExtendedProducerProperties properties = createProducerProperties(); + properties.getExtension().setSync(true); + Binding producerBinding = binder.bindProducer(testTopicName, + output, properties); + DirectFieldAccessor accessor = new DirectFieldAccessor( + extractEndpoint(producerBinding)); + KafkaProducerMessageHandler wrappedInstance = (KafkaProducerMessageHandler) accessor + .getWrappedInstance(); + assertThat(new DirectFieldAccessor(wrappedInstance).getPropertyValue("sync") + .equals(Boolean.TRUE)) + .withFailMessage("Kafka Sync Producer should have been enabled."); + producerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testSendTimeoutExpressionProducerMetadata() throws Exception { + Binder binder = getBinder(createConfigurationProperties()); + DirectChannel output = new DirectChannel(); + String testTopicName = UUID.randomUUID().toString(); + ExtendedProducerProperties properties = createProducerProperties(); + properties.getExtension().setSync(true); + SpelExpressionParser parser = new SpelExpressionParser(); + Expression sendTimeoutExpression = parser.parseExpression("5000"); + properties.getExtension().setSendTimeoutExpression(sendTimeoutExpression); + Binding producerBinding = binder.bindProducer(testTopicName, + output, properties); + DirectFieldAccessor accessor = new DirectFieldAccessor( + extractEndpoint(producerBinding)); + KafkaProducerMessageHandler wrappedInstance = (KafkaProducerMessageHandler) accessor + .getWrappedInstance(); + assertThat(new DirectFieldAccessor(wrappedInstance).getPropertyValue("sendTimeoutExpression") + .equals(sendTimeoutExpression)); + producerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testAutoCreateTopicsDisabledOnBinderStillWorksAsLongAsBrokerCreatesTopic() + throws Exception { + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + configurationProperties.setAutoCreateTopics(false); + Binder binder = getBinder(configurationProperties); + BindingProperties producerBindingProperties = createProducerBindingProperties( + createProducerProperties()); + DirectChannel output = createBindableChannel("output", producerBindingProperties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + + DirectChannel input = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + String testTopicName = "createdByBroker-" + System.currentTimeMillis(); + + Binding producerBinding = binder.bindProducer(testTopicName, + output, producerBindingProperties.getProducer()); + + String testPayload = "foo1-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload)); + + Binding consumerBinding = binder.bindConsumer(testTopicName, + "test", input, consumerProperties); + CountDownLatch latch = new CountDownLatch(1); + AtomicReference> inboundMessageRef = new AtomicReference<>(); + input.subscribe(message1 -> { + try { + inboundMessageRef.set((Message) message1); + } + finally { + latch.countDown(); + } + }); + Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message"); + + assertThat(inboundMessageRef.get()).isNotNull(); + assertThat( + new String(inboundMessageRef.get().getPayload(), StandardCharsets.UTF_8)) + .isEqualTo(testPayload); + + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testAutoConfigureTopicsDisabledSucceedsIfTopicExisting() + throws Throwable { + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + + String testTopicName = "existing" + System.currentTimeMillis(); + invokeCreateTopic(testTopicName, 5, 1); + configurationProperties.setAutoCreateTopics(false); + Binder binder = getBinder(configurationProperties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + + DirectChannel input = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + Binding binding = binder.bindConsumer(testTopicName, "test", + input, consumerProperties); + binding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testPartitionCountIncreasedIfAutoAddPartitionsSet() throws Throwable { + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + + String testTopicName = "existing" + System.currentTimeMillis(); + configurationProperties.setMinPartitionCount(6); + configurationProperties.setAutoAddPartitions(true); + Binder binder = getBinder(configurationProperties); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + DirectChannel input = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + Binding binding = binder.bindConsumer(testTopicName, "test", input, + consumerProperties); + binding.unbind(); + assertThat(invokePartitionSize(testTopicName)).isEqualTo(6); + } + + @Test + @SuppressWarnings("unchecked") + public void testAutoAddPartitionsDisabledSucceedsIfTopicUnderPartitionedAndAutoRebalanceEnabled() + throws Throwable { + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + + String testTopicName = "existing" + System.currentTimeMillis(); + invokeCreateTopic(testTopicName, 1, 1); + configurationProperties.setAutoAddPartitions(false); + Binder binder = getBinder(configurationProperties); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + + DirectChannel input = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + // this consumer must consume from partition 2 + consumerProperties.setInstanceCount(3); + consumerProperties.setInstanceIndex(2); + Binding binding = binder.bindConsumer(testTopicName, "test", input, + consumerProperties); + binding.unbind(); + assertThat(invokePartitionSize(testTopicName)).isEqualTo(1); + } + + @Test + @SuppressWarnings("unchecked") + public void testAutoAddPartitionsDisabledFailsIfTopicUnderPartitionedAndAutoRebalanceDisabled() + throws Throwable { + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + + String testTopicName = "existing" + System.currentTimeMillis(); + invokeCreateTopic(testTopicName, 1, 1); + configurationProperties.setAutoAddPartitions(false); + Binder binder = getBinder(configurationProperties); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + DirectChannel output = createBindableChannel("output", + createConsumerBindingProperties(consumerProperties)); + // this consumer must consume from partition 2 + consumerProperties.setInstanceCount(3); + consumerProperties.setInstanceIndex(2); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + Assertions.assertThatThrownBy(() -> { + Binding binding = binder.bindConsumer(testTopicName, "test", output, + consumerProperties); + if (binding != null) { + binding.unbind(); + } + }).isInstanceOf(ProvisioningException.class); + } + + @Test + @SuppressWarnings("unchecked") + public void testAutoAddPartitionsDisabledSucceedsIfTopicPartitionedCorrectly() + throws Throwable { + Binding binding = null; + try { + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + + String testTopicName = "existing" + System.currentTimeMillis(); + invokeCreateTopic(testTopicName, 6, 1); + configurationProperties.setAutoAddPartitions(false); + Binder binder = getBinder(configurationProperties); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + + DirectChannel input = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + // this consumer must consume from partition 2 + consumerProperties.setInstanceCount(3); + consumerProperties.setInstanceIndex(2); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + + binding = binder.bindConsumer(testTopicName, "test-x", input, + consumerProperties); + + ContainerProperties containerProps = TestUtils.getPropertyValue( + binding, + "lifecycle.messageListenerContainer.containerProperties", + ContainerProperties.class); + TopicPartitionOffset[] listenedPartitions = containerProps.getTopicPartitions(); + assertThat(listenedPartitions).hasSize(2); + assertThat(listenedPartitions).contains( + new TopicPartitionOffset(testTopicName, 2), + new TopicPartitionOffset(testTopicName, 5)); + int partitions = invokePartitionSize(testTopicName); + assertThat(partitions).isEqualTo(6); + } + finally { + if (binding != null) { + binding.unbind(); + } + } + } + + @Test + @SuppressWarnings("unchecked") + public void testPartitionCountNotReduced() throws Throwable { + String testTopicName = "existing" + System.currentTimeMillis(); + + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + + invokeCreateTopic(testTopicName, 6, 1); + configurationProperties.setAutoAddPartitions(true); + Binder binder = getBinder(configurationProperties); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + DirectChannel input = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + Binding binding = binder.bindConsumer(testTopicName, "test", input, + consumerProperties); + binding.unbind(); + + assertThat(partitionSize(testTopicName)).isEqualTo(6); + } + + @Test + @SuppressWarnings("unchecked") + public void testConsumerDefaultDeserializer() throws Throwable { + Binding binding = null; + try { + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + String testTopicName = "existing" + System.currentTimeMillis(); + invokeCreateTopic(testTopicName, 5, 1); + configurationProperties.setAutoCreateTopics(false); + Binder binder = getBinder(configurationProperties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + DirectChannel input = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + binding = binder.bindConsumer(testTopicName, "test", input, + consumerProperties); + DirectFieldAccessor consumerAccessor = new DirectFieldAccessor( + getKafkaConsumer(binding)); + assertThat(consumerAccessor + .getPropertyValue("keyDeserializer") instanceof ByteArrayDeserializer) + .isTrue(); + assertThat(consumerAccessor.getPropertyValue( + "valueDeserializer") instanceof ByteArrayDeserializer).isTrue(); + } + finally { + if (binding != null) { + binding.unbind(); + } + } + } + + @Test + @SuppressWarnings("unchecked") + public void testConsumerCustomDeserializer() throws Exception { + Binding binding = null; + try { + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + Map propertiesToOverride = configurationProperties + .getConfiguration(); + propertiesToOverride.put("key.deserializer", + "org.apache.kafka.common.serialization.StringDeserializer"); + propertiesToOverride.put("value.deserializer", + "org.apache.kafka.common.serialization.LongDeserializer"); + configurationProperties.setConfiguration(propertiesToOverride); + String testTopicName = "existing" + System.currentTimeMillis(); + configurationProperties.setAutoCreateTopics(false); + Binder binder = getBinder(configurationProperties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + DirectChannel input = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + binding = binder.bindConsumer(testTopicName, "test", input, + consumerProperties); + DirectFieldAccessor consumerAccessor = new DirectFieldAccessor( + getKafkaConsumer(binding)); + assertThat(consumerAccessor + .getPropertyValue("keyDeserializer") instanceof StringDeserializer) + .isTrue(); + assertThat(consumerAccessor + .getPropertyValue("valueDeserializer") instanceof LongDeserializer) + .isTrue(); + } + finally { + if (binding != null) { + binding.unbind(); + } + } + } + + @Test + @SuppressWarnings({ "unchecked", "rawtypes" }) + public void testNativeSerializationWithCustomSerializerDeserializer() + throws Exception { + Binding producerBinding = null; + Binding consumerBinding = null; + try { + Integer testPayload = 10; + Message message = MessageBuilder.withPayload(testPayload).build(); + SubscribableChannel moduleOutputChannel = new DirectChannel(); + String testTopicName = "existing" + System.currentTimeMillis(); + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + configurationProperties.setAutoAddPartitions(true); + Binder binder = getBinder(configurationProperties); + QueueChannel moduleInputChannel = new QueueChannel(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setUseNativeEncoding(true); + producerProperties.getExtension().getConfiguration().put("value.serializer", + "org.apache.kafka.common.serialization.IntegerSerializer"); + producerBinding = binder.bindProducer(testTopicName, moduleOutputChannel, + producerProperties); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + consumerProperties.getExtension().getConfiguration().put("value.deserializer", + "org.apache.kafka.common.serialization.IntegerDeserializer"); + consumerProperties.getExtension() + .setStandardHeaders(KafkaConsumerProperties.StandardHeaders.both); + consumerBinding = binder.bindConsumer(testTopicName, "test", + moduleInputChannel, consumerProperties); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = receive(moduleInputChannel, 500); + assertThat(inbound).isNotNull(); + assertThat(inbound.getPayload()).isEqualTo(10); + assertThat(inbound.getHeaders()).doesNotContainKey("contentType"); + assertThat(inbound.getHeaders().getId()).isNotNull(); + assertThat(inbound.getHeaders().getTimestamp()).isNotNull(); + } + finally { + if (producerBinding != null) { + producerBinding.unbind(); + } + if (consumerBinding != null) { + consumerBinding.unbind(); + } + } + } + + private KafkaConsumer getKafkaConsumer(Binding binding) { + DirectFieldAccessor bindingAccessor = new DirectFieldAccessor(binding); + KafkaMessageDrivenChannelAdapter adapter = (KafkaMessageDrivenChannelAdapter) bindingAccessor + .getPropertyValue("lifecycle"); + DirectFieldAccessor adapterAccessor = new DirectFieldAccessor(adapter); + ConcurrentMessageListenerContainer messageListenerContainer = (ConcurrentMessageListenerContainer) adapterAccessor + .getPropertyValue("messageListenerContainer"); + DirectFieldAccessor containerAccessor = new DirectFieldAccessor( + messageListenerContainer); + DefaultKafkaConsumerFactory consumerFactory = (DefaultKafkaConsumerFactory) containerAccessor + .getPropertyValue("consumerFactory"); + return (KafkaConsumer) consumerFactory.createConsumer(); + } + + @Test + @SuppressWarnings({ "unchecked", "rawtypes" }) + public void testNativeSerializationWithCustomSerializerDeserializerBytesPayload() + throws Exception { + Binding producerBinding = null; + Binding consumerBinding = null; + try { + byte[] testPayload = new byte[1]; + Message message = MessageBuilder.withPayload(testPayload) + .setHeader(MessageHeaders.CONTENT_TYPE, "something/funky").build(); + SubscribableChannel moduleOutputChannel = new DirectChannel(); + String testTopicName = "existing" + System.currentTimeMillis(); + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + configurationProperties.setAutoAddPartitions(true); + Binder binder = getBinder(configurationProperties); + ConfigurableApplicationContext context = TestUtils.getPropertyValue(binder, + "binder.applicationContext", ConfigurableApplicationContext.class); + MessagingMessageConverter converter = new MessagingMessageConverter(); + converter.setGenerateMessageId(true); + converter.setGenerateTimestamp(true); + context.getBeanFactory().registerSingleton("testConverter", converter); + QueueChannel moduleInputChannel = new QueueChannel(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setUseNativeEncoding(true); + producerProperties.getExtension().getConfiguration().put( + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + ByteArraySerializer.class.getName()); + producerBinding = binder.bindProducer(testTopicName, moduleOutputChannel, + producerProperties); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + consumerProperties.getExtension().getConfiguration().put( + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + ByteArrayDeserializer.class.getName()); + consumerProperties.getExtension().setConverterBeanName("testConverter"); + consumerBinding = binder.bindConsumer(testTopicName, "test", + moduleInputChannel, consumerProperties); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = receive(moduleInputChannel, 500); + assertThat(inbound).isNotNull(); + assertThat(inbound.getPayload()).isEqualTo(new byte[1]); + assertThat(inbound.getHeaders()).containsKey("contentType"); + assertThat(inbound.getHeaders().get(MessageHeaders.CONTENT_TYPE).toString()) + .isEqualTo("something/funky"); + assertThat(inbound.getHeaders().getId()).isNotNull(); + assertThat(inbound.getHeaders().getTimestamp()).isNotNull(); + } + finally { + if (producerBinding != null) { + producerBinding.unbind(); + } + if (consumerBinding != null) { + consumerBinding.unbind(); + } + } + } + + @Test + @SuppressWarnings("unchecked") + public void testBuiltinSerialization() throws Exception { + Binding producerBinding = null; + Binding consumerBinding = null; + try { + String testPayload = "test"; + Message message = MessageBuilder.withPayload(testPayload) + .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN) + .build(); + + ExtendedProducerProperties producerProperties = createProducerProperties(); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.getExtension().setAutoRebalanceEnabled(false); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + String testTopicName = "existing" + System.currentTimeMillis(); + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + configurationProperties.setAutoAddPartitions(true); + Binder binder = getBinder(configurationProperties); + producerBinding = binder.bindProducer(testTopicName, moduleOutputChannel, + producerProperties); + + consumerBinding = binder.bindConsumer(testTopicName, "test", + moduleInputChannel, consumerProperties); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + CountDownLatch latch = new CountDownLatch(1); + AtomicReference> inboundMessageRef = new AtomicReference<>(); + moduleInputChannel.subscribe(message1 -> { + try { + inboundMessageRef.set((Message) message1); + } + finally { + latch.countDown(); + } + }); + Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message"); + + assertThat(inboundMessageRef.get()).isNotNull(); + assertThat(inboundMessageRef.get().getPayload()).isEqualTo("test".getBytes()); + assertThat(inboundMessageRef.get().getHeaders()).containsEntry("contentType", + MimeTypeUtils.TEXT_PLAIN); + } + finally { + if (producerBinding != null) { + producerBinding.unbind(); + } + if (consumerBinding != null) { + consumerBinding.unbind(); + } + } + } + + /* + * Verify that a consumer configured to handle embedded headers can handle all three + * variants. + */ + @Test + @SuppressWarnings({ "unchecked", "rawtypes" }) + public void testSendAndReceiveWithMixedMode() throws Exception { + KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties(); + binderConfiguration.setHeaders("foo"); + Binder binder = getBinder(binderConfiguration); + DirectChannel moduleOutputChannel1 = new DirectChannel(); + ExtendedProducerProperties producerProperties1 = createProducerProperties(); + producerProperties1.setHeaderMode(HeaderMode.embeddedHeaders); + Binding producerBinding1 = binder.bindProducer("mixed.0", + moduleOutputChannel1, producerProperties1); + + DirectChannel moduleOutputChannel2 = new DirectChannel(); + ExtendedProducerProperties producerProperties2 = createProducerProperties(); + producerProperties2.setHeaderMode(HeaderMode.headers); + Binding producerBinding2 = binder.bindProducer("mixed.0", + moduleOutputChannel2, producerProperties2); + + DirectChannel moduleOutputChannel3 = new DirectChannel(); + ExtendedProducerProperties producerProperties3 = createProducerProperties(); + producerProperties3.setHeaderMode(HeaderMode.none); + Binding producerBinding3 = binder.bindProducer("mixed.0", + moduleOutputChannel3, producerProperties3); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setHeaderMode(HeaderMode.embeddedHeaders); + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + QueueChannel bridged = new QueueChannel(); + BridgeHandler bridge = new BridgeHandler(); + bridge.setOutputChannel(bridged); + moduleInputChannel.subscribe(bridge); + Binding consumerBinding = binder.bindConsumer("mixed.0", "test", + moduleInputChannel, consumerProperties); + Message message = org.springframework.integration.support.MessageBuilder + .withPayload("testSendAndReceiveWithMixedMode".getBytes()) + .setHeader("foo", "bar").build(); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel1.send(message); + moduleOutputChannel2.send(message); + moduleOutputChannel3.send(message); + Message inbound = receive(bridged, 10_000); + assertThat(inbound).isNotNull(); + assertThat(new String((byte[]) inbound.getPayload(), StandardCharsets.UTF_8)) + .isEqualTo("testSendAndReceiveWithMixedMode"); + assertThat(inbound.getHeaders().get("foo")).isEqualTo("bar"); + assertThat(inbound.getHeaders().get(BinderHeaders.NATIVE_HEADERS_PRESENT)) + .isNull(); + inbound = receive(bridged); + assertThat(inbound).isNotNull(); + assertThat(new String((byte[]) inbound.getPayload(), StandardCharsets.UTF_8)) + .isEqualTo("testSendAndReceiveWithMixedMode"); + assertThat(inbound.getHeaders().get("foo")).isEqualTo("bar"); + assertThat(inbound.getHeaders().get(BinderHeaders.NATIVE_HEADERS_PRESENT)) + .isEqualTo(Boolean.TRUE); + inbound = receive(bridged); + assertThat(inbound).isNotNull(); + assertThat(new String((byte[]) inbound.getPayload(), StandardCharsets.UTF_8)) + .isEqualTo("testSendAndReceiveWithMixedMode"); + assertThat(inbound.getHeaders().get("foo")).isNull(); + assertThat(inbound.getHeaders().get(BinderHeaders.NATIVE_HEADERS_PRESENT)) + .isNull(); + + Map consumerProps = KafkaTestUtils.consumerProps( + "testSendAndReceiveWithMixedMode", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + ByteArrayDeserializer.class); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + ByteArrayDeserializer.class); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + Consumer consumer = cf.createConsumer(); + consumer.subscribe(Collections.singletonList("mixed.0")); + + ConsumerRecords records = consumer.poll(10_1000); + Iterator iterator = records.iterator(); + ConsumerRecord record = iterator.next(); + byte[] value = (byte[]) record.value(); + assertThat(value[0] & 0xff).isEqualTo(0xff); + assertThat(record.headers().toArray().length).isEqualTo(0); + record = iterator.next(); + value = (byte[]) record.value(); + assertThat(value[0] & 0xff).isNotEqualTo(0xff); + assertThat(record.headers().toArray().length).isEqualTo(2); + record = iterator.next(); + value = (byte[]) record.value(); + assertThat(value[0] & 0xff).isNotEqualTo(0xff); + assertThat(record.headers().toArray().length).isEqualTo(0); + consumer.close(); + + producerBinding1.unbind(); + producerBinding2.unbind(); + producerBinding3.unbind(); + consumerBinding.unbind(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Test + public void testPolledConsumer() throws Exception { + KafkaTestBinder binder = getBinder(); + PollableSource inboundBindTarget = new DefaultPollableMessageSource( + this.messageConverter); + ExtendedConsumerProperties consumerProps = createConsumerProperties(); + consumerProps.setMultiplex(true); + consumerProps.getExtension().setPollTimeout(1); + Binding> binding = binder.bindPollableConsumer( + "pollable,anotherOne", "group-polledConsumer", inboundBindTarget, + consumerProps); + Map producerProps = KafkaTestUtils + .producerProps(embeddedKafka); + KafkaTemplate template = new KafkaTemplate( + new DefaultKafkaProducerFactory<>(producerProps)); + template.send("pollable", "testPollable"); + boolean polled = inboundBindTarget.poll(m -> { + assertThat(m.getPayload()).isEqualTo("testPollable"); + }); + int n = 0; + while (n++ < 100 && !polled) { + polled = inboundBindTarget.poll(m -> { + assertThat(m.getPayload()).isEqualTo("testPollable".getBytes()); + }); + Thread.sleep(100); + } + assertThat(polled).isTrue(); + + template.send("anotherOne", "testPollable2"); + polled = inboundBindTarget.poll(m -> { + assertThat(m.getPayload()).isEqualTo("testPollable2"); + }); + n = 0; + while (n++ < 100 && !polled) { + polled = inboundBindTarget.poll(m -> { + assertThat(m.getPayload()).isEqualTo("testPollable2".getBytes()); + }); + Thread.sleep(100); + } + assertThat(polled).isTrue(); + // Bind a second pollable consumer GH-521 + consumerProps.getExtension().getConfiguration() + .put(ConsumerConfig.CLIENT_ID_CONFIG, "pollable2"); + PollableSource second = new DefaultPollableMessageSource( + this.messageConverter); + Binding> binding2 = binder.bindPollableConsumer( + "pollable2", "group-polledConsumer2", second, consumerProps); + second.poll(m -> { + }); + binding.unbind(); + binding2.unbind(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Test + public void testPolledConsumerRequeue() throws Exception { + KafkaTestBinder binder = getBinder(); + PollableSource inboundBindTarget = new DefaultPollableMessageSource( + this.messageConverter); + ExtendedConsumerProperties properties = createConsumerProperties(); + Binding> binding = binder.bindPollableConsumer( + "pollableRequeue", "group", inboundBindTarget, properties); + Map producerProps = KafkaTestUtils + .producerProps(embeddedKafka); + KafkaTemplate template = new KafkaTemplate( + new DefaultKafkaProducerFactory<>(producerProps)); + template.send("pollableRequeue", "testPollable"); + try { + boolean polled = false; + int n = 0; + while (n++ < 100 && !polled) { + polled = inboundBindTarget.poll(m -> { + assertThat(m.getPayload()).isEqualTo("testPollable".getBytes()); + throw new RequeueCurrentMessageException(); + }); + } + } + catch (MessageHandlingException e) { + assertThat(e.getCause()).isInstanceOf(RequeueCurrentMessageException.class); + } + boolean polled = inboundBindTarget.poll(m -> { + assertThat(m.getPayload()).isEqualTo("testPollable".getBytes()); + }); + assertThat(polled).isTrue(); + binding.unbind(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Test + public void testPolledConsumerWithDlq() throws Exception { + KafkaTestBinder binder = getBinder(); + PollableSource inboundBindTarget = new DefaultPollableMessageSource( + this.messageConverter); + ExtendedConsumerProperties properties = createConsumerProperties(); + properties.getExtension().setPollTimeout(1); + properties.setMaxAttempts(2); + properties.setBackOffInitialInterval(0); + properties.getExtension().setEnableDlq(true); + Map producerProps = KafkaTestUtils + .producerProps(embeddedKafka); + Binding> binding = binder.bindPollableConsumer( + "pollableDlq", "group-pcWithDlq", inboundBindTarget, properties); + KafkaTemplate template = new KafkaTemplate( + new DefaultKafkaProducerFactory<>(producerProps)); + template.send("pollableDlq", "testPollableDLQ"); + try { + int n = 0; + while (n++ < 100) { + inboundBindTarget.poll(m -> { + throw new RuntimeException("test DLQ"); + }); + Thread.sleep(100); + } + } + catch (MessageHandlingException e) { + assertThat(e.getCause().getMessage()).isEqualTo("test DLQ"); + } + Map consumerProps = KafkaTestUtils.consumerProps("dlq", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + ConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + Consumer consumer = cf.createConsumer(); + embeddedKafka.consumeFromAnEmbeddedTopic(consumer, + "error.pollableDlq.group-pcWithDlq"); + ConsumerRecord deadLetter = KafkaTestUtils.getSingleRecord(consumer, + "error.pollableDlq.group-pcWithDlq"); + assertThat(deadLetter).isNotNull(); + assertThat(deadLetter.value()).isEqualTo("testPollableDLQ"); + binding.unbind(); + consumer.close(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Test + public void testTopicPatterns() throws Exception { + try (AdminClient admin = AdminClient.create( + Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, + embeddedKafka.getBrokersAsString()))) { + admin.createTopics(Collections + .singletonList(new NewTopic("topicPatterns.1", 1, (short) 1))).all() + .get(); + Binder binder = getBinder(); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.getExtension().setDestinationIsPattern(true); + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + final CountDownLatch latch = new CountDownLatch(1); + final AtomicReference topic = new AtomicReference<>(); + moduleInputChannel.subscribe(m -> { + topic.set(m.getHeaders().get(KafkaHeaders.RECEIVED_TOPIC, String.class)); + latch.countDown(); + }); + Binding consumerBinding = binder.bindConsumer( + "topicPatterns\\..*", "testTopicPatterns", moduleInputChannel, + consumerProperties); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory( + KafkaTestUtils.producerProps(embeddedKafka)); + KafkaTemplate template = new KafkaTemplate(pf); + template.send("topicPatterns.1", "foo"); + assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(topic.get()).isEqualTo("topicPatterns.1"); + consumerBinding.unbind(); + pf.destroy(); + } + } + + @Test + public void testSameTopicCannotBeProvisionedAgain() throws Throwable { + CountDownLatch latch = new CountDownLatch(1); + try (AdminClient admin = AdminClient.create( + Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, + embeddedKafka.getBrokersAsString()))) { + admin.createTopics(Collections + .singletonList(new NewTopic("fooUniqueTopic", 1, (short) 1))).all() + .get(); + try { + admin.createTopics(Collections + .singletonList(new NewTopic("fooUniqueTopic", 1, (short) 1))) + .all().get(); + fail("Expecting TopicExistsException"); + } + catch (Exception ex) { + assertThat(ex.getCause() instanceof TopicExistsException).isTrue(); + latch.countDown(); + } + latch.await(1, TimeUnit.SECONDS); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testResetOffsets() throws Exception { + Binding producerBinding = null; + Binding consumerBinding = null; + try { + String testPayload = "test"; + + ExtendedProducerProperties producerProperties = createProducerProperties(); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setConcurrency(2); + consumerProperties.setInstanceCount(5); // 10 partitions across 2 threads + consumerProperties.getExtension().setResetOffsets(true); + + DirectChannel moduleInputChannel = createBindableChannel("input", + createConsumerBindingProperties(consumerProperties)); + + String testTopicName = "existing" + System.currentTimeMillis(); + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + configurationProperties.setAutoAddPartitions(true); + Binder binder = getBinder(configurationProperties); + producerBinding = binder.bindProducer(testTopicName, moduleOutputChannel, + producerProperties); + + consumerBinding = binder.bindConsumer(testTopicName, "testReset", + moduleInputChannel, consumerProperties); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + IntStream.range(0, 10).forEach(i -> moduleOutputChannel.send(MessageBuilder.withPayload(testPayload) + .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN) + .setHeader(KafkaHeaders.PARTITION_ID, i) + .build())); + CountDownLatch latch1 = new CountDownLatch(10); + CountDownLatch latch2 = new CountDownLatch(20); + AtomicReference> inboundMessageRef = new AtomicReference<>(); + AtomicInteger received = new AtomicInteger(); + moduleInputChannel.subscribe(message1 -> { + try { + inboundMessageRef.set((Message) message1); + } + finally { + received.incrementAndGet(); + latch1.countDown(); + latch2.countDown(); + } + }); + assertThat(latch1.await(10, TimeUnit.SECONDS)).as("Failed to receive messages").isTrue(); + consumerBinding.unbind(); + consumerBinding = binder.bindConsumer(testTopicName, "testReset", + moduleInputChannel, consumerProperties); + assertThat(latch2.await(10, TimeUnit.SECONDS)).as("Failed to receive message").isTrue(); + binder.bindConsumer(testTopicName + "-x", "testReset", + moduleInputChannel, consumerProperties).unbind(); // cause another rebalance + assertThat(received.get()).as("Unexpected reset").isEqualTo(20); + + assertThat(inboundMessageRef.get()).isNotNull(); + assertThat(inboundMessageRef.get().getPayload()).isEqualTo("test".getBytes()); + assertThat(inboundMessageRef.get().getHeaders()).containsEntry("contentType", + MimeTypeUtils.TEXT_PLAIN); + } + finally { + if (producerBinding != null) { + producerBinding.unbind(); + } + if (consumerBinding != null) { + consumerBinding.unbind(); + } + } + } + + @Test + @SuppressWarnings("unchecked") + public void testRecordMetadata() throws Exception { + Binding producerBinding = null; + try { + String testPayload = "test"; + + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.getExtension().setRecordMetadataChannel("metaChannel"); + QueueChannel metaChannel = new QueueChannel(); + + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + + String testTopicName = "existing" + System.currentTimeMillis(); + KafkaTestBinder binder = getBinder(); + ((GenericApplicationContext) binder.getApplicationContext()).registerBean("metaChannel", + MessageChannel.class, () -> metaChannel); + producerBinding = binder.bindProducer(testTopicName, moduleOutputChannel, + producerProperties); + moduleOutputChannel + .send(new GenericMessage<>("foo", Collections.singletonMap(KafkaHeaders.PARTITION_ID, 0))); + Message sendResult = metaChannel.receive(10_000); + assertThat(sendResult).isNotNull(); + RecordMetadata meta = sendResult.getHeaders().get(KafkaHeaders.RECORD_METADATA, RecordMetadata.class); + assertThat(meta).isNotNull() + .hasFieldOrPropertyWithValue("offset", 0L); + assertThat(meta.topic()).isEqualTo(testTopicName); + assertThat(meta.partition()).isEqualTo(0); + } + finally { + if (producerBinding != null) { + producerBinding.unbind(); + } + } + } + + @Test + @SuppressWarnings("unchecked") + public void testMessageKeyInPayload() throws Exception { + Binding producerBinding = null; + try { + String testPayload = "test"; + + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.getExtension() + .setMessageKeyExpression(spelExpressionParser.parseExpression("payload.field.bytes")); + DirectChannel moduleOutputChannel = createBindableChannel("output", + createProducerBindingProperties(producerProperties)); + + String testTopicName = "existing" + System.currentTimeMillis(); + KafkaTestBinder binder = getBinder(); + producerBinding = binder.bindProducer(testTopicName, moduleOutputChannel, + producerProperties); + moduleOutputChannel.addInterceptor(new ChannelInterceptor() { + + @Override + public Message preSend(Message message, MessageChannel channel) { + assertThat(message.getHeaders() + .get(KafkaExpressionEvaluatingInterceptor.MESSAGE_KEY_HEADER)) + .isEqualTo("foo".getBytes()); + return message; + } + + }); + moduleOutputChannel.send( + new GenericMessage<>(new Pojo("foo"), Collections.singletonMap(KafkaHeaders.PARTITION_ID, 0))); + } + finally { + if (producerBinding != null) { + producerBinding.unbind(); + } + } + } + + @Test + public void testInternalHeadersNotPropagated() throws Exception { + testInternalHeadersNotPropagatedGuts("propagate.1", null, null); + } + + @Test + public void testInternalHeadersNotPropagatedCustomHeader() throws Exception { + testInternalHeadersNotPropagatedGuts("propagate.2", new String[] { "foo", "*" }, null); + } + + @Test + public void testInternalHeadersNotPropagatedCustomMapper() throws Exception { + testInternalHeadersNotPropagatedGuts("propagate.3", null, new BinderHeaderMapper("*")); + } + + public void testInternalHeadersNotPropagatedGuts(String name, String[] headerPatterns, + KafkaHeaderMapper mapper) throws Exception { + + KafkaTestBinder binder; + if (mapper == null) { + binder = getBinder(); + } + else { + KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties(); + binderConfiguration.setHeaderMapperBeanName("headerMapper"); + + KafkaTopicProvisioner kafkaTopicProvisioner = new KafkaTopicProvisioner( + binderConfiguration, new TestKafkaProperties(), null); + try { + kafkaTopicProvisioner.afterPropertiesSet(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + binder = new KafkaTestBinder(binderConfiguration, kafkaTopicProvisioner); + ((GenericApplicationContext) binder.getApplicationContext()).registerBean("headerMapper", + KafkaHeaderMapper.class, () -> mapper); + } + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.getExtension().setHeaderPatterns(headerPatterns); + + DirectChannel output = createBindableChannel("output", createProducerBindingProperties(producerProperties)); + output.setBeanName(name + ".out"); + Binding producerBinding = binder.bindProducer(name + ".1", output, producerProperties); + + QueueChannel input = new QueueChannel(); + input.setBeanName(name + ".in"); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + Binding consumerBinding = binder.bindConsumer(name + ".0", name, input, consumerProperties); + Map producerProps = KafkaTestUtils.producerProps(embeddedKafka); + KafkaTemplate template = new KafkaTemplate(new DefaultKafkaProducerFactory<>(producerProps)); + template.send(MessageBuilder.withPayload("internalHeaderPropagation") + .setHeader(KafkaHeaders.TOPIC, name + ".0") + .setHeader("someHeader", "someValue") + .build()); + + Message consumed = input.receive(10_000); + if (headerPatterns != null) { + consumed = MessageBuilder.fromMessage(consumed).setHeader(headerPatterns[0], "bar").build(); + } + output.send(consumed); + + Map consumerProps = KafkaTestUtils.consumerProps(name, "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + Consumer consumer = cf.createConsumer(); + consumer.assign(Collections.singletonList(new TopicPartition(name + ".1", 0))); + ConsumerRecords records = consumer.poll(Duration.ofSeconds(10)); + assertThat(records.count()).isEqualTo(1); + ConsumerRecord received = records.iterator().next(); + assertThat(received.value()).isEqualTo("internalHeaderPropagation".getBytes()); + Header header = received.headers().lastHeader(BinderHeaders.NATIVE_HEADERS_PRESENT); + assertThat(header).isNull(); + header = received.headers().lastHeader(IntegrationMessageHeaderAccessor.DELIVERY_ATTEMPT); + assertThat(header).isNull(); + header = received.headers().lastHeader(MessageHeaders.ID); + assertThat(header).isNull(); + header = received.headers().lastHeader(MessageHeaders.TIMESTAMP); + assertThat(header).isNull(); + assertThat(received.headers().lastHeader("someHeader")).isNotNull(); + if (headerPatterns != null) { + assertThat(received.headers().lastHeader(headerPatterns[0])).isNotNull(); + } + + producerBinding.unbind(); + consumerBinding.unbind(); + consumer.close(); + } + + @Test + public void testNoBrokerOverride() throws Exception { + Binder binder = getBinder(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.getExtension().getConfiguration().put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "foo"); + BindingProperties outputBindingProperties = createProducerBindingProperties(producerProperties); + DirectChannel moduleOutputChannel = createBindableChannel("output", outputBindingProperties); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.getExtension().getConfiguration().put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "foo"); + BindingProperties consumerBindingProperties = createConsumerBindingProperties(consumerProperties); + DirectChannel moduleInputChannel = createBindableChannel("input", consumerBindingProperties); + + assertThatExceptionOfType(BinderException.class).isThrownBy(() -> binder.bindProducer("foo.bar", + moduleOutputChannel, outputBindingProperties.getProducer())) + .withCauseExactlyInstanceOf(IllegalStateException.class); + assertThatExceptionOfType(BinderException.class).isThrownBy(() -> binder.bindConsumer("foo.bar", + "testSendAndReceive", moduleInputChannel, consumerProperties)) + .withCauseExactlyInstanceOf(IllegalStateException.class); + } + + private final class FailingInvocationCountingMessageHandler + implements MessageHandler { + + private volatile int invocationCount; + + private final LinkedHashMap> receivedMessages = new LinkedHashMap<>(); + + private final CountDownLatch latch; + + private FailingInvocationCountingMessageHandler(int latchSize) { + latch = new CountDownLatch(latchSize); + } + + private FailingInvocationCountingMessageHandler() { + this(1); + } + + @Override + public void handleMessage(Message message) throws MessagingException { + invocationCount++; + Long offset = message.getHeaders() + .get(KafkaBinderTests.this.getKafkaOffsetHeaderKey(), Long.class); + // using the offset as key allows to ensure that we don't store duplicate + // messages on retry + if (!receivedMessages.containsKey(offset)) { + receivedMessages.put(offset, message); + latch.countDown(); + } + throw new RuntimeException("fail"); + } + + public LinkedHashMap> getReceivedMessages() { + return receivedMessages; + } + + public int getInvocationCount() { + return invocationCount; + } + + public CountDownLatch getLatch() { + return latch; + } + + } + + public static class Pojo { + + private String field; + + public Pojo() { + super(); + } + + public Pojo(String field) { + this.field = field; + } + + public String getField() { + return this.field; + } + + public void setField(String field) { + this.field = field; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderUnitTests.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderUnitTests.java new file mode 100644 index 000000000..234e32eb0 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderUnitTests.java @@ -0,0 +1,319 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.Deserializer; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.cloud.stream.binder.Binding; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.cloud.stream.config.ListenerContainerCustomizer; +import org.springframework.cloud.stream.provisioning.ConsumerDestination; +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.test.util.TestUtils; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.listener.AbstractMessageListenerContainer; +import org.springframework.messaging.MessageChannel; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.willAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +/** + * @author Gary Russell + * @since 1.2.2 + * + */ +public class KafkaBinderUnitTests { + + @Test + public void testPropertyOverrides() throws Exception { + KafkaProperties kafkaProperties = new TestKafkaProperties(); + KafkaBinderConfigurationProperties binderConfigurationProperties = new KafkaBinderConfigurationProperties( + kafkaProperties); + KafkaTopicProvisioner provisioningProvider = new KafkaTopicProvisioner( + binderConfigurationProperties, kafkaProperties, null); + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder( + binderConfigurationProperties, provisioningProvider); + KafkaConsumerProperties consumerProps = new KafkaConsumerProperties(); + ExtendedConsumerProperties ecp = new ExtendedConsumerProperties( + consumerProps); + Method method = KafkaMessageChannelBinder.class.getDeclaredMethod( + "createKafkaConsumerFactory", boolean.class, String.class, + ExtendedConsumerProperties.class, String.class, String.class); + method.setAccessible(true); + + // test default for anon + Object factory = method.invoke(binder, true, "foo-1", ecp, "foo.consumer", "foo"); + Map configs = TestUtils.getPropertyValue(factory, "configs", Map.class); + assertThat(configs.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)) + .isEqualTo("latest"); + + // test default for named + factory = method.invoke(binder, false, "foo-2", ecp, "foo.consumer", "foo"); + configs = TestUtils.getPropertyValue(factory, "configs", Map.class); + assertThat(configs.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)) + .isEqualTo("earliest"); + + // binder level setting + binderConfigurationProperties.setConfiguration(Collections + .singletonMap(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest")); + factory = method.invoke(binder, false, "foo-3", ecp, "foo.consumer", "foo"); + configs = TestUtils.getPropertyValue(factory, "configs", Map.class); + assertThat(configs.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)) + .isEqualTo("latest"); + + // consumer level setting + consumerProps.setConfiguration(Collections + .singletonMap(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")); + factory = method.invoke(binder, false, "foo-4", ecp, "foo.consumer", "foo"); + configs = TestUtils.getPropertyValue(factory, "configs", Map.class); + assertThat(configs.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)) + .isEqualTo("earliest"); + } + + @Test + public void testMergedConsumerProperties() { + KafkaProperties bootProps = new TestKafkaProperties(); + bootProps.getConsumer().getProperties() + .put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "bar"); + KafkaBinderConfigurationProperties props = new KafkaBinderConfigurationProperties( + bootProps); + assertThat(props.mergedConsumerConfiguration() + .get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)).isEqualTo("bar"); + props.getConfiguration().put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "baz"); + assertThat(props.mergedConsumerConfiguration() + .get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)).isEqualTo("baz"); + props.getConsumerProperties().put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "qux"); + assertThat(props.mergedConsumerConfiguration() + .get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)).isEqualTo("qux"); + } + + @Test + public void testMergedProducerProperties() { + KafkaProperties bootProps = new TestKafkaProperties(); + bootProps.getProducer().getProperties().put(ProducerConfig.RETRIES_CONFIG, "bar"); + KafkaBinderConfigurationProperties props = new KafkaBinderConfigurationProperties( + bootProps); + assertThat(props.mergedProducerConfiguration().get(ProducerConfig.RETRIES_CONFIG)) + .isEqualTo("bar"); + props.getConfiguration().put(ProducerConfig.RETRIES_CONFIG, "baz"); + assertThat(props.mergedProducerConfiguration().get(ProducerConfig.RETRIES_CONFIG)) + .isEqualTo("baz"); + props.getProducerProperties().put(ProducerConfig.RETRIES_CONFIG, "qux"); + assertThat(props.mergedProducerConfiguration().get(ProducerConfig.RETRIES_CONFIG)) + .isEqualTo("qux"); + } + + @Test + public void testOffsetResetWithGroupManagementEarliest() throws Exception { + testOffsetResetWithGroupManagement(true, true, "foo-100", + "testOffsetResetWithGroupManagementEarliest"); + } + + @Test + public void testOffsetResetWithGroupManagementLatest() throws Throwable { + testOffsetResetWithGroupManagement(false, true, "foo-101", + "testOffsetResetWithGroupManagementLatest"); + } + + @Test + public void testOffsetResetWithManualAssignmentEarliest() throws Exception { + testOffsetResetWithGroupManagement(true, false, "foo-102", + "testOffsetResetWithManualAssignmentEarliest"); + } + + @Test + public void testOffsetResetWithGroupManualAssignmentLatest() throws Throwable { + testOffsetResetWithGroupManagement(false, false, "foo-103", + "testOffsetResetWithGroupManualAssignmentLatest"); + } + + private void testOffsetResetWithGroupManagement(final boolean earliest, + boolean groupManage, String topic, String group) throws Exception { + + final List partitions = new ArrayList<>(); + partitions.add(new TopicPartition(topic, 0)); + partitions.add(new TopicPartition(topic, 1)); + KafkaBinderConfigurationProperties configurationProperties = new KafkaBinderConfigurationProperties( + new TestKafkaProperties()); + KafkaTopicProvisioner provisioningProvider = mock(KafkaTopicProvisioner.class); + ConsumerDestination dest = mock(ConsumerDestination.class); + given(dest.getName()).willReturn(topic); + given(provisioningProvider.provisionConsumerDestination(anyString(), anyString(), + any())).willReturn(dest); + final AtomicInteger part = new AtomicInteger(); + willAnswer(i -> { + return partitions.stream().map(p -> new PartitionInfo(topic, + part.getAndIncrement(), null, null, null)) + .collect(Collectors.toList()); + }).given(provisioningProvider).getPartitionsForTopic(anyInt(), anyBoolean(), + any(), any()); + @SuppressWarnings("unchecked") + final Consumer consumer = mock(Consumer.class); + final CountDownLatch latch = new CountDownLatch(1); + willAnswer(i -> { + try { + Thread.sleep(100); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return new ConsumerRecords<>(Collections.emptyMap()); + }).given(consumer).poll(any(Duration.class)); + willAnswer(i -> { + ((org.apache.kafka.clients.consumer.ConsumerRebalanceListener) i + .getArgument(1)).onPartitionsAssigned(partitions); + latch.countDown(); + return null; + }).given(consumer).subscribe(eq(Collections.singletonList(topic)), any()); + willAnswer(i -> { + latch.countDown(); + return null; + }).given(consumer).seekToBeginning(any()); + willAnswer(i -> { + latch.countDown(); + return null; + }).given(consumer).seekToEnd(any()); + class Customizer implements ListenerContainerCustomizer> { + + @Override + public void configure(AbstractMessageListenerContainer container, String destinationName, + String group) { + + container.getContainerProperties().setMissingTopicsFatal(false); + } + + } + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder( + configurationProperties, provisioningProvider, new Customizer(), null) { + + @Override + protected ConsumerFactory createKafkaConsumerFactory(boolean anonymous, + String consumerGroup, + ExtendedConsumerProperties consumerProperties, String beanName, String destination) { + + return new ConsumerFactory() { + + @Override + public Consumer createConsumer() { + return consumer; + } + + @Override + public Consumer createConsumer(String arg0) { + return consumer; + } + + @Override + public Consumer createConsumer(String arg0, + String arg1) { + return consumer; + } + + @Override + public Consumer createConsumer(String groupId, + String clientIdPrefix, String clientIdSuffix) { + return consumer; + } + + @Override + public boolean isAutoCommit() { + return false; + } + + @Override + public Map getConfigurationProperties() { + Map props = new HashMap<>(); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, + earliest ? "earliest" : "latest"); + props.put(ConsumerConfig.GROUP_ID_CONFIG, group); + return props; + } + + @Override + public Deserializer getKeyDeserializer() { + return null; + } + + @Override + public Deserializer getValueDeserializer() { + return null; + } + + }; + } + + }; + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + binder.setApplicationContext(context); + MessageChannel channel = new DirectChannel(); + KafkaConsumerProperties extension = new KafkaConsumerProperties(); + extension.setResetOffsets(true); + extension.setAutoRebalanceEnabled(groupManage); + ExtendedConsumerProperties consumerProperties = new ExtendedConsumerProperties<>( + extension); + consumerProperties.setInstanceCount(1); + Binding messageChannelBinding = binder.bindConsumer(topic, group, + channel, consumerProperties); + assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue(); + @SuppressWarnings("unchecked") + ArgumentCaptor> captor = ArgumentCaptor.forClass(Set.class); + if (earliest) { + verify(consumer).seekToBeginning(captor.capture()); + } + else { + verify(consumer).seekToEnd(captor.capture()); + } + assertThat(captor.getValue()).containsExactlyInAnyOrderElementsOf(partitions); + messageChannelBinding.unbind(); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaTestBinder.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaTestBinder.java new file mode 100644 index 000000000..c09249698 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaTestBinder.java @@ -0,0 +1,92 @@ +/* + * Copyright 2015-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.cloud.stream.binder.kafka.utils.DlqDestinationResolver; +import org.springframework.cloud.stream.binder.kafka.utils.DlqPartitionFunction; +import org.springframework.cloud.stream.provisioning.ConsumerDestination; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Configuration; +import org.springframework.integration.config.EnableIntegration; +import org.springframework.kafka.support.LoggingProducerListener; +import org.springframework.kafka.support.ProducerListener; + +/** + * Test support class for {@link KafkaMessageChannelBinder}. + * + * @author Eric Bottard + * @author Marius Bogoevici + * @author David Turanski + * @author Gary Russell + * @author Soby Chacko + */ +public class KafkaTestBinder extends AbstractKafkaTestBinder { + + KafkaTestBinder(KafkaBinderConfigurationProperties binderConfiguration, + KafkaTopicProvisioner kafkaTopicProvisioner) { + + this(binderConfiguration, kafkaTopicProvisioner, null, null); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + KafkaTestBinder(KafkaBinderConfigurationProperties binderConfiguration, + KafkaTopicProvisioner kafkaTopicProvisioner, DlqPartitionFunction dlqPartitionFunction, + DlqDestinationResolver dlqDestinationResolver) { + + try { + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder( + binderConfiguration, kafkaTopicProvisioner, null, null, null, dlqPartitionFunction, dlqDestinationResolver) { + + /* + * Some tests use multiple instance indexes for the same topic; we need to + * make the error infrastructure beans unique. + */ + @Override + protected String errorsBaseName(ConsumerDestination destination, + String group, + ExtendedConsumerProperties consumerProperties) { + return super.errorsBaseName(destination, group, consumerProperties) + + "-" + consumerProperties.getInstanceIndex(); + } + + }; + + ProducerListener producerListener = new LoggingProducerListener(); + binder.setProducerListener(producerListener); + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext( + Config.class); + setApplicationContext(context); + binder.setApplicationContext(context); + binder.afterPropertiesSet(); + this.setPollableConsumerBinder(binder); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Configuration + @EnableIntegration + static class Config { + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaTransactionTests.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaTransactionTests.java new file mode 100644 index 000000000..518d01fa7 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaTransactionTests.java @@ -0,0 +1,122 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.Collections; + +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.TopicPartition; +import org.junit.ClassRule; +import org.junit.Test; +import org.mockito.InOrder; + +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; +import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.test.util.TestUtils; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.util.concurrent.SettableListenableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.willReturn; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +/** + * @author Gary Russell + * @since 2.0 + * + */ +public class KafkaTransactionTests { + + @ClassRule + public static final EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1) + .brokerProperty("transaction.state.log.replication.factor", "1") + .brokerProperty("transaction.state.log.min.isr", "1"); + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Test + public void testProducerRunsInTx() { + KafkaProperties kafkaProperties = new TestKafkaProperties(); + kafkaProperties.setBootstrapServers(Collections + .singletonList(embeddedKafka.getEmbeddedKafka().getBrokersAsString())); + KafkaBinderConfigurationProperties configurationProperties = new KafkaBinderConfigurationProperties( + kafkaProperties); + configurationProperties.getTransaction().setTransactionIdPrefix("foo-"); + configurationProperties.getTransaction().getProducer().setUseNativeEncoding(true); + KafkaTopicProvisioner provisioningProvider = new KafkaTopicProvisioner( + configurationProperties, kafkaProperties, null); + provisioningProvider.setMetadataRetryOperations(new RetryTemplate()); + final Producer mockProducer = mock(Producer.class); + given(mockProducer.send(any(), any())).willReturn(new SettableListenableFuture<>()); + + KafkaProducerProperties extension1 = configurationProperties + .getTransaction().getProducer().getExtension(); + extension1.getConfiguration().put(ProducerConfig.RETRIES_CONFIG, "1"); + extension1.getConfiguration().put(ProducerConfig.ACKS_CONFIG, "all"); + + willReturn(Collections.singletonList(new TopicPartition("foo", 0))) + .given(mockProducer).partitionsFor(anyString()); + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder( + configurationProperties, provisioningProvider) { + + @Override + protected DefaultKafkaProducerFactory getProducerFactory( + String transactionIdPrefix, + ExtendedProducerProperties producerProperties, String beanName, String destination) { + DefaultKafkaProducerFactory producerFactory = spy( + super.getProducerFactory(transactionIdPrefix, + producerProperties, beanName, destination)); + willReturn(mockProducer).given(producerFactory).createProducer("foo-"); + return producerFactory; + } + + }; + GenericApplicationContext applicationContext = new GenericApplicationContext(); + applicationContext.refresh(); + binder.setApplicationContext(applicationContext); + DirectChannel channel = new DirectChannel(); + KafkaProducerProperties extension = new KafkaProducerProperties(); + ExtendedProducerProperties properties = new ExtendedProducerProperties<>( + extension); + binder.bindProducer("foo", channel, properties); + channel.send(new GenericMessage<>("foo".getBytes())); + InOrder inOrder = inOrder(mockProducer); + inOrder.verify(mockProducer).beginTransaction(); + inOrder.verify(mockProducer).send(any(ProducerRecord.class), any(Callback.class)); + inOrder.verify(mockProducer).commitTransaction(); + inOrder.verify(mockProducer).close(any()); + inOrder.verifyNoMoreInteractions(); + assertThat(TestUtils.getPropertyValue(channel, + "dispatcher.theOneHandler.useNativeEncoding", Boolean.class)).isTrue(); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/RawKafkaPartitionTestSupport.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/RawKafkaPartitionTestSupport.java new file mode 100644 index 000000000..4d98f495c --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/RawKafkaPartitionTestSupport.java @@ -0,0 +1,39 @@ +/* + * Copyright 2014-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import org.springframework.cloud.stream.binder.PartitionKeyExtractorStrategy; +import org.springframework.cloud.stream.binder.PartitionSelectorStrategy; +import org.springframework.messaging.Message; + +/** + * @author Marius Bogoevici + */ +public class RawKafkaPartitionTestSupport + implements PartitionKeyExtractorStrategy, PartitionSelectorStrategy { + + @Override + public int selectPartition(Object key, int divisor) { + return ((byte[]) key)[0] % divisor; + } + + @Override + public Object extractKey(Message message) { + return message.getPayload(); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/TestKafkaProperties.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/TestKafkaProperties.java new file mode 100644 index 000000000..0c64b8bc2 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/TestKafkaProperties.java @@ -0,0 +1,41 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; + +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; + +/** + * Test {@link KafkaProperties} initialized in the same way as the + * {@code KafkaBinderEnvironmentPostProcessor} initializes the properties. + * + * @author Gary Russell + * @since 2.1 + * + */ +public class TestKafkaProperties extends KafkaProperties { + + public TestKafkaProperties() { + getConsumer().setKeyDeserializer(ByteArrayDeserializer.class); + getConsumer().setValueDeserializer(ByteArrayDeserializer.class); + getProducer().setKeySerializer(ByteArraySerializer.class); + getProducer().setValueSerializer(ByteArraySerializer.class); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/bootstrap/KafkaBinderBootstrapTest.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/bootstrap/KafkaBinderBootstrapTest.java new file mode 100644 index 000000000..757c371a1 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/bootstrap/KafkaBinderBootstrapTest.java @@ -0,0 +1,52 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.bootstrap; + +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; + +/** + * @author Marius Bogoevici + */ +public class KafkaBinderBootstrapTest { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, 10); + + @Test + public void testKafkaBinderConfiguration() throws Exception { + ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder( + SimpleApplication.class).web(WebApplicationType.NONE).run( + "--spring.cloud.stream.kafka.binder.brokers=" + + embeddedKafka.getEmbeddedKafka().getBrokersAsString(), + "--spring.cloud.stream.kafka.binder.zkNodes=" + embeddedKafka + .getEmbeddedKafka().getZookeeperConnectionString()); + applicationContext.close(); + } + + @SpringBootApplication + static class SimpleApplication { + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/bootstrap/KafkaBinderCustomHealthCheckTests.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/bootstrap/KafkaBinderCustomHealthCheckTests.java new file mode 100644 index 000000000..d11288714 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/bootstrap/KafkaBinderCustomHealthCheckTests.java @@ -0,0 +1,72 @@ +/* + * Copyright 2022-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.bootstrap; + +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.beans.factory.NoSuchBeanDefinitionException; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealth; +import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealthIndicator; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; + +/** + * @author Soby Chacko + */ +public class KafkaBinderCustomHealthCheckTests { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, 10); + + @Test + public void testCustomHealthIndicatorIsActivated() { + ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder( + CustomHealthCheckApplication.class).web(WebApplicationType.NONE).run( + "--spring.cloud.stream.kafka.binder.brokers=" + + embeddedKafka.getEmbeddedKafka().getBrokersAsString()); + final KafkaBinderHealth kafkaBinderHealth = applicationContext.getBean(KafkaBinderHealth.class); + assertThat(kafkaBinderHealth).isInstanceOf(CustomHealthIndicator.class); + assertThatThrownBy(() -> applicationContext.getBean(KafkaBinderHealthIndicator.class)).isInstanceOf(NoSuchBeanDefinitionException.class); + applicationContext.close(); + } + + @SpringBootApplication + static class CustomHealthCheckApplication { + + @Bean + public CustomHealthIndicator kafkaBinderHealthIndicator() { + return new CustomHealthIndicator(); + } + } + + static class CustomHealthIndicator implements KafkaBinderHealth { + + @Override + public Health health() { + return null; + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/bootstrap/KafkaBinderMeterRegistryTest.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/bootstrap/KafkaBinderMeterRegistryTest.java new file mode 100644 index 000000000..b3d2488dd --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/bootstrap/KafkaBinderMeterRegistryTest.java @@ -0,0 +1,159 @@ +/* + * Copyright 2019-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.bootstrap; + +import java.util.Map; +import java.util.function.Function; + +import io.micrometer.core.instrument.MeterRegistry; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.condition.EmbeddedKafkaCondition; +import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.kafka.test.utils.KafkaTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +/** + * @author Soby Chacko + */ +@EmbeddedKafka(count = 1, controlledShutdown = true, partitions = 10, topics = "outputTopic") +public class KafkaBinderMeterRegistryTest { + + private static EmbeddedKafkaBroker embeddedKafka; + + private static Consumer consumer; + + @BeforeAll + public static void setup() { + embeddedKafka = EmbeddedKafkaCondition.getBroker(); + + Map consumerProps = KafkaTestUtils.consumerProps("group", "false", + embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer, "outputTopic"); + } + + @AfterAll + public static void tearDown() { + consumer.close(); + } + + @Test + public void testMetricsWithSingleBinder() throws Exception { + ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder(SimpleApplication.class) + .web(WebApplicationType.NONE) + .run("--spring.cloud.stream.bindings.uppercase-in-0.destination=inputTopic", + "--spring.cloud.stream.bindings.uppercase-in-0.group=inputGroup", + "--spring.cloud.stream.bindings.uppercase-out-0.destination=outputTopic", + "--spring.cloud.stream.kafka.binder.brokers" + "=" + + embeddedKafka.getBrokersAsString()); + + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("inputTopic"); + template.sendDefault("foo"); + + // Forcing the retrieval of the data on the outbound so that the producer factory has + // a chance to add the micrometer listener properly. Only on the first send, binder's + // internal KafkaTemplate adds the Micrometer listener (using the producer factory). + KafkaTestUtils.getSingleRecord(consumer, "outputTopic"); + + final MeterRegistry meterRegistry = applicationContext.getBean(MeterRegistry.class); + assertMeterRegistry(meterRegistry); + applicationContext.close(); + } + + @Test + public void testMetricsWithMultiBinders() { + ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder(SimpleApplication.class) + .web(WebApplicationType.NONE) + .run("--spring.cloud.stream.bindings.uppercase-in-0.destination=inputTopic", + "--spring.cloud.stream.bindings.uppercase-in-0.group=inputGroup", + "--spring.cloud.stream.bindings.uppercase-in-0.binder=kafka1", + "--spring.cloud.stream.bindings.uppercase-out-0.destination=outputTopic", + "--spring.cloud.stream.bindings.uppercase-out-0.binder=kafka2", + "--spring.cloud.stream.binders.kafka1.type=kafka", + "--spring.cloud.stream.binders.kafka2.type=kafka", + "--spring.cloud.stream.binders.kafka1.environment" + + ".spring.cloud.stream.kafka.binder.brokers" + "=" + + embeddedKafka.getBrokersAsString(), + "--spring.cloud.stream.binders.kafka2.environment" + + ".spring.cloud.stream.kafka.binder.brokers" + "=" + + embeddedKafka.getBrokersAsString()); + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("inputTopic"); + template.sendDefault("foo"); + + // Forcing the retrieval of the data on the outbound so that the producer factory has + // a chance to add the micrometer listener properly. Only on the first send, binder's + // internal KafkaTemplate adds the Micrometer listener (using the producer factory). + KafkaTestUtils.getSingleRecord(consumer, "outputTopic"); + + final MeterRegistry meterRegistry = applicationContext.getBean(MeterRegistry.class); + assertMeterRegistry(meterRegistry); + applicationContext.close(); + } + + private void assertMeterRegistry(MeterRegistry meterRegistry) { + assertThat(meterRegistry).isNotNull(); + + // assert kafka binder metrics + assertThat(meterRegistry.get("spring.cloud.stream.binder.kafka.offset") + .tag("group", "inputGroup") + .tag("topic", "inputTopic").gauge().value()).isNotNull(); + + // assert consumer metrics + assertThatCode(() -> meterRegistry.get("kafka.consumer.fetch.manager.fetch.total").meter()).doesNotThrowAnyException(); + + // assert producer metrics + assertThatCode(() -> meterRegistry.get("kafka.producer.io.ratio").meter()).doesNotThrowAnyException(); + } + + @SpringBootApplication + static class SimpleApplication { + + @Bean + public Function uppercase() { + return String::toUpperCase; + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderActuatorTests.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderActuatorTests.java new file mode 100644 index 000000000..f722dc510 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderActuatorTests.java @@ -0,0 +1,190 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.integration; + +import java.util.List; +import java.util.Map; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.binder.MeterBinder; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.FilteredClassLoader; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.Binding; +import org.springframework.cloud.stream.binder.PollableMessageSource; +import org.springframework.cloud.stream.binding.BindingService; +import org.springframework.cloud.stream.config.ConsumerEndpointCustomizer; +import org.springframework.cloud.stream.config.ListenerContainerCustomizer; +import org.springframework.cloud.stream.config.MessageSourceCustomizer; +import org.springframework.cloud.stream.config.ProducerMessageHandlerCustomizer; +import org.springframework.cloud.stream.messaging.Processor; +import org.springframework.cloud.stream.messaging.Sink; +import org.springframework.context.annotation.Bean; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.inbound.KafkaMessageSource; +import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.listener.AbstractMessageListenerContainer; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.messaging.MessageChannel; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Artem Bilan + * @author Oleg Zhurakousky + * @author Jon Schneider + * @author Gary Russell + * + * @since 2.0 + */ +@RunWith(SpringRunner.class) +// @checkstyle:off +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = "spring.cloud.stream.bindings.input.group=" + + KafkaBinderActuatorTests.TEST_CONSUMER_GROUP) +// @checkstyle:on +@DirtiesContext +public class KafkaBinderActuatorTests { + + static final String TEST_CONSUMER_GROUP = "testGroup-actuatorTests"; + + private static final String KAFKA_BROKERS_PROPERTY = "spring.kafka.bootstrap-servers"; + + @ClassRule + public static EmbeddedKafkaRule kafkaEmbedded = new EmbeddedKafkaRule(1, true); + + @BeforeClass + public static void setup() { + System.setProperty(KAFKA_BROKERS_PROPERTY, + kafkaEmbedded.getEmbeddedKafka().getBrokersAsString()); + } + + @AfterClass + public static void clean() { + System.clearProperty(KAFKA_BROKERS_PROPERTY); + } + + @Autowired + private MeterRegistry meterRegistry; + + @Autowired + private KafkaTemplate kafkaTemplate; + + @Test + public void testKafkaBinderMetricsExposed() { + this.kafkaTemplate.send(Sink.INPUT, null, "foo".getBytes()); + this.kafkaTemplate.flush(); + + assertThat(this.meterRegistry.get("spring.cloud.stream.binder.kafka.offset") + .tag("group", TEST_CONSUMER_GROUP).tag("topic", Sink.INPUT).gauge() + .value()).isGreaterThan(0); + } + + @Test + public void testKafkaBinderMetricsWhenNoMicrometer() { + new ApplicationContextRunner().withUserConfiguration(KafkaMetricsTestConfig.class) + .withClassLoader(new FilteredClassLoader("io.micrometer.core")) + .run(context -> { + assertThat(context.getBeanNamesForType(MeterRegistry.class)) + .isEmpty(); + assertThat(context.getBeanNamesForType(MeterBinder.class)).isEmpty(); + + DirectFieldAccessor channelBindingServiceAccessor = new DirectFieldAccessor( + context.getBean(BindingService.class)); + @SuppressWarnings("unchecked") + Map>> consumerBindings = + (Map>>) channelBindingServiceAccessor + .getPropertyValue("consumerBindings"); + assertThat(new DirectFieldAccessor( + consumerBindings.get("input").get(0)).getPropertyValue( + "lifecycle.messageListenerContainer.beanName")) + .isEqualTo("setByCustomizer:input"); + assertThat(new DirectFieldAccessor( + consumerBindings.get("input").get(0)).getPropertyValue( + "lifecycle.beanName")) + .isEqualTo("setByCustomizer:input"); + assertThat(new DirectFieldAccessor( + consumerBindings.get("source").get(0)).getPropertyValue( + "lifecycle.beanName")) + .isEqualTo("setByCustomizer:source"); + + @SuppressWarnings("unchecked") + Map> producerBindings = + (Map>) channelBindingServiceAccessor + .getPropertyValue("producerBindings"); + + assertThat(new DirectFieldAccessor( + producerBindings.get("output")).getPropertyValue( + "lifecycle.beanName")) + .isEqualTo("setByCustomizer:output"); + }); + } + + @EnableBinding({ Processor.class, PMS.class }) + @EnableAutoConfiguration + public static class KafkaMetricsTestConfig { + + @Bean + public ListenerContainerCustomizer> containerCustomizer() { + return (c, q, g) -> c.setBeanName("setByCustomizer:" + q); + } + + @Bean + public MessageSourceCustomizer> sourceCustomizer() { + return (s, q, g) -> s.setBeanName("setByCustomizer:" + q); + } + + @Bean + public ConsumerEndpointCustomizer> consumerCustomizer() { + return (p, q, g) -> p.setBeanName("setByCustomizer:" + q); + } + + @Bean + public ProducerMessageHandlerCustomizer> handlerCustomizer() { + return (handler, destinationName) -> handler.setBeanName("setByCustomizer:" + destinationName); + } + + @StreamListener(Sink.INPUT) + public void process(@SuppressWarnings("unused") String payload) throws InterruptedException { + // Artificial slow listener to emulate consumer lag + Thread.sleep(1000); + } + + } + + public interface PMS { + + @Input + PollableMessageSource source(); + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderExtendedPropertiesTest.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderExtendedPropertiesTest.java new file mode 100644 index 000000000..db2064c0d --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaBinderExtendedPropertiesTest.java @@ -0,0 +1,237 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.integration; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.common.TopicPartition; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.Output; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.Binder; +import org.springframework.cloud.stream.binder.BinderFactory; +import org.springframework.cloud.stream.binder.ConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder; +import org.springframework.cloud.stream.binder.ProducerProperties; +import org.springframework.cloud.stream.binder.kafka.KafkaBindingRebalanceListener; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; +import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.SubscribableChannel; +import org.springframework.messaging.handler.annotation.SendTo; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Soby Chacko + * @author Gary Russell + */ +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { + "spring.cloud.stream.kafka.bindings.standard-out.producer.configuration.key.serializer=FooSerializer.class", + "spring.cloud.stream.kafka.default.producer.configuration.key.serializer=BarSerializer.class", + "spring.cloud.stream.kafka.default.producer.configuration.value.serializer=BarSerializer.class", + "spring.cloud.stream.kafka.bindings.standard-in.consumer.configuration.key.serializer=FooSerializer.class", + "spring.cloud.stream.kafka.default.consumer.configuration.key.serializer=BarSerializer.class", + "spring.cloud.stream.kafka.default.consumer.configuration.value.serializer=BarSerializer.class", + "spring.cloud.stream.kafka.default.producer.configuration.foo=bar", + "spring.cloud.stream.kafka.bindings.standard-out.producer.configuration.foo=" + + "bindingSpecificPropertyShouldWinOverDefault", + "spring.cloud.stream.kafka.default.consumer.ackEachRecord=true", + "spring.cloud.stream.kafka.bindings.custom-in.consumer.ackEachRecord=false" }) +@DirtiesContext +public class KafkaBinderExtendedPropertiesTest { + + private static final String KAFKA_BROKERS_PROPERTY = "spring.cloud.stream.kafka.binder.brokers"; + + @ClassRule + public static EmbeddedKafkaRule kafkaEmbedded = new EmbeddedKafkaRule(1, true); + + @BeforeClass + public static void setup() { + System.setProperty(KAFKA_BROKERS_PROPERTY, + kafkaEmbedded.getEmbeddedKafka().getBrokersAsString()); + } + + @AfterClass + public static void clean() { + System.clearProperty(KAFKA_BROKERS_PROPERTY); + } + + @Autowired + private ConfigurableApplicationContext context; + + @Test + public void testKafkaBinderExtendedProperties() throws Exception { + + BinderFactory binderFactory = context.getBeanFactory() + .getBean(BinderFactory.class); + Binder kafkaBinder = binderFactory + .getBinder("kafka", MessageChannel.class); + + KafkaProducerProperties kafkaProducerProperties = (KafkaProducerProperties) ((ExtendedPropertiesBinder) kafkaBinder) + .getExtendedProducerProperties("standard-out"); + + // binding "standard-out" gets FooSerializer defined on the binding itself + // and BarSerializer through default property. + assertThat(kafkaProducerProperties.getConfiguration().get("key.serializer")) + .isEqualTo("FooSerializer.class"); + assertThat(kafkaProducerProperties.getConfiguration().get("value.serializer")) + .isEqualTo("BarSerializer.class"); + + assertThat(kafkaProducerProperties.getConfiguration().get("foo")) + .isEqualTo("bindingSpecificPropertyShouldWinOverDefault"); + + // @checkstyle:off + KafkaConsumerProperties kafkaConsumerProperties = (KafkaConsumerProperties) ((ExtendedPropertiesBinder) kafkaBinder) + .getExtendedConsumerProperties("standard-in"); + // @checkstyle:on + // binding "standard-in" gets FooSerializer defined on the binding itself + // and BarSerializer through default property. + assertThat(kafkaConsumerProperties.getConfiguration().get("key.serializer")) + .isEqualTo("FooSerializer.class"); + assertThat(kafkaConsumerProperties.getConfiguration().get("value.serializer")) + .isEqualTo("BarSerializer.class"); + + // @checkstyle:off + KafkaProducerProperties customKafkaProducerProperties = (KafkaProducerProperties) ((ExtendedPropertiesBinder) kafkaBinder) + .getExtendedProducerProperties("custom-out"); + // @checkstyle:on + + // binding "standard-out" gets BarSerializer and BarSerializer for + // key.serializer/value.serializer through default properties. + assertThat(customKafkaProducerProperties.getConfiguration().get("key.serializer")) + .isEqualTo("BarSerializer.class"); + assertThat( + customKafkaProducerProperties.getConfiguration().get("value.serializer")) + .isEqualTo("BarSerializer.class"); + + // through default properties. + assertThat(customKafkaProducerProperties.getConfiguration().get("foo")) + .isEqualTo("bar"); + + // @checkstyle:off + KafkaConsumerProperties customKafkaConsumerProperties = (KafkaConsumerProperties) ((ExtendedPropertiesBinder) kafkaBinder) + .getExtendedConsumerProperties("custom-in"); + // @checkstyle:on + // binding "standard-in" gets BarSerializer and BarSerializer for + // key.serializer/value.serializer through default properties. + assertThat(customKafkaConsumerProperties.getConfiguration().get("key.serializer")) + .isEqualTo("BarSerializer.class"); + assertThat( + customKafkaConsumerProperties.getConfiguration().get("value.serializer")) + .isEqualTo("BarSerializer.class"); + + assertThat(kafkaConsumerProperties.isAckEachRecord()).isEqualTo(true); + assertThat(customKafkaConsumerProperties.isAckEachRecord()).isEqualTo(false); + + RebalanceListener rebalanceListener = context.getBean(RebalanceListener.class); + assertThat(rebalanceListener.latch.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(rebalanceListener.bindings.keySet()).contains("standard-in", + "custom-in"); + assertThat(rebalanceListener.bindings.values()).containsExactly(Boolean.TRUE, + Boolean.TRUE); + } + + @EnableBinding(CustomBindingForExtendedPropertyTesting.class) + @EnableAutoConfiguration + public static class KafkaMetricsTestConfig { + + @StreamListener("standard-in") + @SendTo("standard-out") + public String process(String payload) { + return payload; + } + + @StreamListener("custom-in") + @SendTo("custom-out") + public String processCustom(String payload) { + return payload; + } + + @Bean + public RebalanceListener rebalanceListener() { + return new RebalanceListener(); + } + + } + + interface CustomBindingForExtendedPropertyTesting { + + @Input("standard-in") + SubscribableChannel standardIn(); + + @Output("standard-out") + MessageChannel standardOut(); + + @Input("custom-in") + SubscribableChannel customIn(); + + @Output("custom-out") + MessageChannel customOut(); + + } + + public static class RebalanceListener implements KafkaBindingRebalanceListener { + + private final Map bindings = new HashMap<>(); + + private final CountDownLatch latch = new CountDownLatch(2); + + @Override + public void onPartitionsRevokedBeforeCommit(String bindingName, + Consumer consumer, Collection partitions) { + + } + + @Override + public void onPartitionsRevokedAfterCommit(String bindingName, + Consumer consumer, Collection partitions) { + + } + + @Override + public void onPartitionsAssigned(String bindingName, Consumer consumer, + Collection partitions, boolean initial) { + + this.bindings.put(bindingName, initial); + this.latch.countDown(); + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaConfigCustomizationTests.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaConfigCustomizationTests.java new file mode 100644 index 000000000..660931a1e --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaConfigCustomizationTests.java @@ -0,0 +1,184 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.integration; + +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerInterceptor; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerInterceptor; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.TopicPartition; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.stream.binder.kafka.config.ConsumerConfigCustomizer; +import org.springframework.cloud.stream.binder.kafka.config.ProducerConfigCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Soby Chacko + * + * Based on: https://github.com/spring-projects/spring-kafka/issues/897#issuecomment-466060097 + */ +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = {"spring.cloud.function.definition=process", + "spring.cloud.stream.bindings.process-in-0.group=KafkaConfigCustomizationTests.group"}) +@DirtiesContext +public class KafkaConfigCustomizationTests { + + private static final String KAFKA_BROKERS_PROPERTY = "spring.cloud.stream.kafka.binder.brokers"; + + @ClassRule + public static EmbeddedKafkaRule kafkaEmbedded = new EmbeddedKafkaRule(1, true); + + static final CountDownLatch countDownLatch = new CountDownLatch(2); + + @BeforeClass + public static void setup() { + System.setProperty(KAFKA_BROKERS_PROPERTY, + kafkaEmbedded.getEmbeddedKafka().getBrokersAsString()); + } + + @AfterClass + public static void clean() { + System.clearProperty(KAFKA_BROKERS_PROPERTY); + } + + @Test + public void testBothConsumerAndProducerConfigsCanBeCustomized() throws InterruptedException { + Map producerProps = KafkaTestUtils + .producerProps(kafkaEmbedded.getEmbeddedKafka()); + KafkaTemplate template = new KafkaTemplate<>( + new DefaultKafkaProducerFactory<>(producerProps)); + template.send("process-in-0", "test-foo"); + template.flush(); + assertThat(countDownLatch.await(10, TimeUnit.SECONDS)).isTrue(); + } + + @SpringBootApplication + public static class ConfigCustomizerTestConfig { + + @Bean + public Function process() { + return payload -> payload; + } + + @Bean + public ConsumerConfigCustomizer consumerConfigCustomizer() { + return (consumerProperties, binding, destination) -> { + assertThat(binding).isEqualTo("process-in-0"); + assertThat(destination).isEqualTo("process-in-0"); + consumerProperties.put(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, MyConsumerInterceptor.class.getName()); + consumerProperties.put("foo.bean", foo()); + }; + } + + @Bean + public ProducerConfigCustomizer producerConfigCustomizer() { + return (producerProperties, binding, destination) -> { + assertThat(binding).isEqualTo("process-out-0"); + assertThat(destination).isEqualTo("process-out-0"); + producerProperties.put(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MyProducerInterceptor.class.getName()); + producerProperties.put("foo.bean", foo()); + }; + } + + @Bean + public Foo foo() { + return new Foo(); + } + } + + public static class Foo { + + public void foo(String what) { + KafkaConfigCustomizationTests.countDownLatch.countDown(); + } + + } + + public static class MyConsumerInterceptor implements ConsumerInterceptor { + + private Foo foo; + + @Override + public void configure(Map configs) { + this.foo = (Foo) configs.get("foo.bean"); + } + + @Override + public ConsumerRecords onConsume(ConsumerRecords records) { + this.foo.foo("consumer interceptor"); + return records; + } + + @Override + public void onCommit(Map offsets) { + + } + + @Override + public void close() { + } + + } + + public static class MyProducerInterceptor implements ProducerInterceptor { + + private Foo foo; + + @Override + public void configure(Map configs) { + this.foo = (Foo) configs.get("foo.bean"); + } + + @Override + public ProducerRecord onSend(ProducerRecord record) { + this.foo.foo("producer interceptor"); + return record; + } + + @Override + public void onAcknowledgement(RecordMetadata metadata, Exception exception) { + } + + @Override + public void close() { + } + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaNullConverterTest.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaNullConverterTest.java new file mode 100644 index 000000000..3015b10f8 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaNullConverterTest.java @@ -0,0 +1,135 @@ +/* + * Copyright 2016-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.integration; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.Output; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.KafkaNull; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.handler.annotation.Payload; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Aldo Sinanaj + * @author Gary Russell + */ +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { + "spring.kafka.consumer.auto-offset-reset=earliest" }) +@DirtiesContext +@Ignore +public class KafkaNullConverterTest { + + private static final String KAFKA_BROKERS_PROPERTY = "spring.kafka.bootstrap-servers"; + + @Autowired + private MessageChannel kafkaNullOutput; + + @Autowired + private MessageChannel kafkaNullInput; + + @Autowired + private KafkaNullConverterTestConfig config; + + @ClassRule + public static EmbeddedKafkaRule kafkaEmbedded = new EmbeddedKafkaRule(1, true); + + @BeforeClass + public static void setup() { + System.setProperty(KAFKA_BROKERS_PROPERTY, + kafkaEmbedded.getEmbeddedKafka().getBrokersAsString()); + } + + @AfterClass + public static void clean() { + System.clearProperty(KAFKA_BROKERS_PROPERTY); + } + + @Test + public void testKafkaNullConverterOutput() throws InterruptedException { + this.kafkaNullOutput.send(new GenericMessage<>(KafkaNull.INSTANCE)); + + assertThat(this.config.countDownLatchOutput.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(this.config.outputPayload).isNull(); + } + + @Test + public void testKafkaNullConverterInput() throws InterruptedException { + this.kafkaNullInput.send(new GenericMessage<>(KafkaNull.INSTANCE)); + + assertThat(this.config.countDownLatchInput.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(this.config.inputPayload).isNull(); + } + + @TestConfiguration + @EnableBinding(KafkaNullTestChannels.class) + public static class KafkaNullConverterTestConfig { + + final CountDownLatch countDownLatchOutput = new CountDownLatch(1); + + final CountDownLatch countDownLatchInput = new CountDownLatch(1); + + volatile byte[] outputPayload = new byte[0]; + + volatile byte[] inputPayload = new byte[0]; + + @KafkaListener(id = "foo", topics = "kafkaNullOutput") + public void listen(@Payload(required = false) byte[] in) { + this.outputPayload = in; + countDownLatchOutput.countDown(); + } + + @StreamListener("kafkaNullInput") + public void inputListen(@Payload(required = false) byte[] payload) { + this.inputPayload = payload; + countDownLatchInput.countDown(); + } + + } + + public interface KafkaNullTestChannels { + + @Input + MessageChannel kafkaNullInput(); + + @Output + MessageChannel kafkaNullOutput(); + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaRetryDlqBinderOrContainerTests.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaRetryDlqBinderOrContainerTests.java new file mode 100644 index 000000000..2002ee773 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaRetryDlqBinderOrContainerTests.java @@ -0,0 +1,117 @@ +/* + * Copyright 2021-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.integration; + +import java.util.function.BiFunction; +import java.util.function.Consumer; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.TopicPartition; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.stream.binder.Binding; +import org.springframework.cloud.stream.binder.kafka.ListenerContainerWithDlqAndRetryCustomizer; +import org.springframework.cloud.stream.binding.BindingsLifecycleController; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.KafkaOperations; +import org.springframework.kafka.listener.AbstractMessageListenerContainer; +import org.springframework.kafka.listener.CommonErrorHandler; +import org.springframework.kafka.listener.ConsumerRecordRecoverer; +import org.springframework.kafka.listener.DeadLetterPublishingRecoverer; +import org.springframework.kafka.listener.DefaultErrorHandler; +import org.springframework.kafka.support.ExponentialBackOffWithMaxRetries; +import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.lang.Nullable; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.util.backoff.BackOff; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +/** + * @author Gary Russell + */ +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { + "spring.cloud.function.definition=retryInBinder;retryInContainer", + "spring.cloud.stream.bindings.retryInBinder-in-0.group=foo", + "spring.cloud.stream.bindings.retryInContainer-in-0.group=bar", + "spring.cloud.stream.kafka.bindings.retryInBinder-in-0.consumer.enable-dlq=true", + "spring.cloud.stream.kafka.bindings.retryInContainer-in-0.consumer.enable-dlq=true"}) +@EmbeddedKafka(bootstrapServersProperty = "spring.kafka.bootstrap-servers") +@DirtiesContext +public class KafkaRetryDlqBinderOrContainerTests { + + @Test + public void retryAndDlqInRightPlace(@Autowired BindingsLifecycleController controller) { + Binding retryInBinder = controller.queryState("retryInBinder-in-0"); + assertThat(KafkaTestUtils.getPropertyValue(retryInBinder, "lifecycle.retryTemplate")).isNotNull(); + assertThat(KafkaTestUtils.getPropertyValue(retryInBinder, + "lifecycle.messageListenerContainer.commonErrorHandler")).isNull(); + Binding retryInContainer = controller.queryState("retryInContainer-in-0"); + assertThat(KafkaTestUtils.getPropertyValue(retryInContainer, "lifecycle.retryTemplate")).isNull(); + assertThat(KafkaTestUtils.getPropertyValue(retryInContainer, + "lifecycle.messageListenerContainer.commonErrorHandler")).isInstanceOf(CommonErrorHandler.class); + assertThat(KafkaTestUtils.getPropertyValue(retryInContainer, + "lifecycle.messageListenerContainer.commonErrorHandler.failureTracker.backOff")) + .isInstanceOf(ExponentialBackOffWithMaxRetries.class); + } + + @SpringBootApplication + public static class ConfigCustomizerTestConfig { + + @Bean + public Consumer retryInBinder() { + return str -> { }; + } + + @Bean + public Consumer retryInContainer() { + return str -> { }; + } + + @Bean + ListenerContainerWithDlqAndRetryCustomizer cust() { + return new ListenerContainerWithDlqAndRetryCustomizer() { + + @Override + public void configure(AbstractMessageListenerContainer container, String destinationName, + String group, + BiFunction, Exception, TopicPartition> dlqDestinationResolver, + @Nullable BackOff backOff) { + + if (destinationName.contains("Container")) { + ConsumerRecordRecoverer dlpr = new DeadLetterPublishingRecoverer(mock(KafkaOperations.class), + dlqDestinationResolver); + container.setCommonErrorHandler(new DefaultErrorHandler(dlpr, backOff)); + } + } + + @Override + public boolean retryAndDlqInBinding(String destinationName, String group) { + return !destinationName.contains("Container"); + } + + }; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/ProducerOnlyTransactionTests.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/ProducerOnlyTransactionTests.java new file mode 100644 index 000000000..41e14f26d --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/ProducerOnlyTransactionTests.java @@ -0,0 +1,154 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.integration; + +import java.util.Map; + +import kafka.server.KafkaConfig; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.IsolationLevel; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.BeanCreationException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.binder.BinderFactory; +import org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder; +import org.springframework.cloud.stream.messaging.Source; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.kafka.transaction.KafkaTransactionManager; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.AbstractPlatformTransactionManager; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @since 2.1.4 + * + */ +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { + "spring.cloud.stream.kafka.binder.transaction.transaction-id-prefix=tx.", + "spring.cloud.stream.kafka.binder.transaction.producer.configuration.retries=99", + "spring.cloud.stream.kafka.binder.transaction.producer.configuration.acks=all"}) +@DirtiesContext +public class ProducerOnlyTransactionTests { + + private static final String KAFKA_BROKERS_PROPERTY = "spring.cloud.stream.kafka.binder.brokers"; + + @ClassRule + public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, "output") + .brokerProperty(KafkaConfig.TransactionsTopicReplicationFactorProp(), "1") + .brokerProperty(KafkaConfig.TransactionsTopicMinISRProp(), "1"); + + @Autowired + private Sender sender; + + @Autowired + private MessageChannel output; + + @BeforeClass + public static void setup() { + System.setProperty(KAFKA_BROKERS_PROPERTY, + embeddedKafka.getEmbeddedKafka().getBrokersAsString()); + } + + @AfterClass + public static void clean() { + System.clearProperty(KAFKA_BROKERS_PROPERTY); + } + + @Test + public void testProducerTx() { + this.sender.DoInTransaction(this.output); + assertThat(this.sender.isInTx()).isTrue(); + Map props = KafkaTestUtils.consumerProps("consumeTx", "false", + embeddedKafka.getEmbeddedKafka()); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_COMMITTED.name().toLowerCase()); + Consumer consumer = new KafkaConsumer<>(props); + embeddedKafka.getEmbeddedKafka().consumeFromAllEmbeddedTopics(consumer); + ConsumerRecord record = KafkaTestUtils.getSingleRecord(consumer, "output"); + assertThat(record.value()).isEqualTo("foo".getBytes()); + } + + @EnableBinding(Source.class) + @EnableAutoConfiguration + @EnableTransactionManagement + public static class Config { + + @Bean + public PlatformTransactionManager transactionManager(BinderFactory binders) { + try { + ProducerFactory pf = ((KafkaMessageChannelBinder) binders.getBinder(null, + MessageChannel.class)).getTransactionalProducerFactory(); + KafkaTransactionManager tm = new KafkaTransactionManager<>(pf); + tm.setTransactionSynchronization(AbstractPlatformTransactionManager.SYNCHRONIZATION_ON_ACTUAL_TRANSACTION); + return tm; + } + catch (BeanCreationException e) { // needed to avoid other tests in this package failing when there is no binder + return null; + } + } + + @Bean + public Sender sender() { + return new Sender(); + } + + } + + public static class Sender { + + private boolean isInTx; + + @Transactional + public void DoInTransaction(MessageChannel output) { + this.isInTx = TransactionSynchronizationManager.isActualTransactionActive(); + output.send(new GenericMessage<>("foo")); + } + + public boolean isInTx() { + return this.isInTx; + } + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/BaseKafkaBinderTopicPropertiesUpdateTest.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/BaseKafkaBinderTopicPropertiesUpdateTest.java new file mode 100644 index 000000000..bfaee22c1 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/BaseKafkaBinderTopicPropertiesUpdateTest.java @@ -0,0 +1,88 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.integration.topic.configs; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.runner.RunWith; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.Output; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.SubscribableChannel; +import org.springframework.messaging.handler.annotation.SendTo; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; + +/** + * @author Heiko Does + */ +@RunWith(SpringRunner.class) +@SpringBootTest( + classes = BaseKafkaBinderTopicPropertiesUpdateTest.TopicAutoConfigsTestConfig.class, + webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { + "spring.cloud.stream.kafka.bindings.standard-out.producer.topic.properties.retention.ms=9001", + "spring.cloud.stream.kafka.default.producer.topic.properties.retention.ms=-1", + "spring.cloud.stream.kafka.bindings.standard-in.consumer.topic.properties.retention.ms=9001", + "spring.cloud.stream.kafka.default.consumer.topic.properties.retention.ms=-1" +}) +@DirtiesContext +public abstract class BaseKafkaBinderTopicPropertiesUpdateTest { + + private static final String KAFKA_BROKERS_PROPERTY = "spring.cloud.stream.kafka.binder.brokers"; + + @ClassRule + public static EmbeddedKafkaRule kafkaEmbedded = new EmbeddedKafkaRule(1, true, "standard-in", "standard-out"); + + @BeforeClass + public static void setup() { + System.setProperty(KAFKA_BROKERS_PROPERTY, + kafkaEmbedded.getEmbeddedKafka().getBrokersAsString()); + } + + @AfterClass + public static void clean() { + System.clearProperty(KAFKA_BROKERS_PROPERTY); + } + + @EnableBinding(CustomBindingForTopicPropertiesUpdateTesting.class) + @EnableAutoConfiguration + public static class TopicAutoConfigsTestConfig { + + @StreamListener("standard-in") + @SendTo("standard-out") + public String process(String payload) { + return payload; + } + } + + interface CustomBindingForTopicPropertiesUpdateTesting { + + @Input("standard-in") + SubscribableChannel standardIn(); + + @Output("standard-out") + MessageChannel standardOut(); + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/DisabledKafkaBinderTopicPropertiesUpdateTest.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/DisabledKafkaBinderTopicPropertiesUpdateTest.java new file mode 100644 index 000000000..89393faf1 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/DisabledKafkaBinderTopicPropertiesUpdateTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.integration.topic.configs; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.Config; +import org.apache.kafka.clients.admin.DescribeConfigsResult; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.config.ConfigResource; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Taras Danylchuk + */ +public class DisabledKafkaBinderTopicPropertiesUpdateTest extends BaseKafkaBinderTopicPropertiesUpdateTest { + + @Test + public void testKafkaBinderShouldNotUpdateTopicConfigurationOnDisabledFeature() throws Exception { + Map adminClientConfig = new HashMap<>(); + adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaEmbedded.getEmbeddedKafka().getBrokersAsString()); + AdminClient adminClient = AdminClient.create(adminClientConfig); + ConfigResource standardInConfigResource = new ConfigResource(ConfigResource.Type.TOPIC, "standard-in"); + ConfigResource standardOutConfigResource = new ConfigResource(ConfigResource.Type.TOPIC, "standard-out"); + DescribeConfigsResult describeConfigsResult = adminClient.describeConfigs(Arrays + .asList(standardInConfigResource, standardOutConfigResource)); + KafkaFuture> kafkaFuture = describeConfigsResult.all(); + Map configResourceConfigMap = kafkaFuture.get(3, TimeUnit.SECONDS); + Config standardInTopicConfig = configResourceConfigMap.get(standardInConfigResource); + assertThat(standardInTopicConfig.get("retention.ms").value()).isEqualTo("604800000"); + + Config standardOutTopicConfig = configResourceConfigMap.get(standardOutConfigResource); + assertThat(standardOutTopicConfig.get("retention.ms").value()).isEqualTo("604800000"); + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/KafkaBinderTopicPropertiesUpdateTest.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/KafkaBinderTopicPropertiesUpdateTest.java new file mode 100644 index 000000000..95b7f9cbc --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/KafkaBinderTopicPropertiesUpdateTest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.integration.topic.configs; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.Config; +import org.apache.kafka.clients.admin.DescribeConfigsResult; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.config.ConfigResource; +import org.junit.Test; + +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Heiko Does + */ +@TestPropertySource(properties = "spring.cloud.stream.kafka.binder.autoAlterTopics=true") +public class KafkaBinderTopicPropertiesUpdateTest extends BaseKafkaBinderTopicPropertiesUpdateTest { + + @Test + public void testKafkaBinderUpdateTopicConfiguration() throws Exception { + Map adminClientConfig = new HashMap<>(); + adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaEmbedded.getEmbeddedKafka().getBrokersAsString()); + AdminClient adminClient = AdminClient.create(adminClientConfig); + ConfigResource standardInConfigResource = new ConfigResource(ConfigResource.Type.TOPIC, "standard-in"); + ConfigResource standardOutConfigResource = new ConfigResource(ConfigResource.Type.TOPIC, "standard-out"); + DescribeConfigsResult describeConfigsResult = adminClient.describeConfigs(Arrays + .asList(standardInConfigResource, standardOutConfigResource)); + KafkaFuture> kafkaFuture = describeConfigsResult.all(); + Map configResourceConfigMap = kafkaFuture.get(3, TimeUnit.SECONDS); + Config standardInTopicConfig = configResourceConfigMap.get(standardInConfigResource); + assertThat(standardInTopicConfig.get("retention.ms").value()).isEqualTo("9001"); + + Config standardOutTopicConfig = configResourceConfigMap.get(standardOutConfigResource); + assertThat(standardOutTopicConfig.get("retention.ms").value()).isEqualTo("9001"); + } +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/ConsumerProducerTransactionTests.java b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/ConsumerProducerTransactionTests.java new file mode 100644 index 000000000..4f49b2061 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration2/ConsumerProducerTransactionTests.java @@ -0,0 +1,203 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.integration2; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import kafka.server.KafkaConfig; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.Output; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.config.ListenerContainerCustomizer; +import org.springframework.cloud.stream.messaging.Processor; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.AbstractMessageListenerContainer; +import org.springframework.kafka.listener.DefaultAfterRollbackProcessor; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.kafka.transaction.KafkaAwareTransactionManager; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.SubscribableChannel; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.backoff.FixedBackOff; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; + +/** + * @author Gary Russell + * @since 3.0 + * + */ +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { + "spring.kafka.consumer.properties.isolation.level=read_committed", + "spring.kafka.consumer.enable-auto-commit=false", + "spring.kafka.consumer.auto-offset-reset=earliest", + "spring.cloud.stream.bindings.input.destination=consumer.producer.txIn", + "spring.cloud.stream.bindings.input.group=consumer.producer.tx", + "spring.cloud.stream.bindings.input.consumer.max-attempts=1", + "spring.cloud.stream.kafka.bindings.input2.consumer.transaction-manager=tm", + "spring.cloud.stream.kafka.bindings.output2.producer.transaction-manager=tm", + "spring.cloud.stream.bindings.output.destination=consumer.producer.txOut", + "spring.cloud.stream.kafka.binder.transaction.transaction-id-prefix=tx.", + "spring.cloud.stream.kafka.binder.transaction.producer.configuration.retries=99", + "spring.cloud.stream.kafka.binder.transaction.producer.configuration.acks=all"}) +@DirtiesContext +public class ConsumerProducerTransactionTests { + + private static final String KAFKA_BROKERS_PROPERTY = "spring.cloud.stream.kafka.binder.brokers"; + + @ClassRule + public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, "consumer.producer.txOut") + .brokerProperty(KafkaConfig.TransactionsTopicReplicationFactorProp(), "1") + .brokerProperty(KafkaConfig.TransactionsTopicMinISRProp(), "1"); + + @Autowired + private Config config; + + @BeforeClass + public static void setup() { + System.setProperty(KAFKA_BROKERS_PROPERTY, + embeddedKafka.getEmbeddedKafka().getBrokersAsString()); + System.setProperty("spring.kafka.bootstrap-servers", + embeddedKafka.getEmbeddedKafka().getBrokersAsString()); + } + + @AfterClass + public static void clean() { + System.clearProperty(KAFKA_BROKERS_PROPERTY); + System.clearProperty("spring.kafka.bootstrap-servers"); + } + + @Test + public void testProducerRunsInConsumerTransaction() throws InterruptedException { + assertThat(this.config.latch.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(this.config.outs).containsExactlyInAnyOrder("ONE", "THREE"); + } + + @Test + public void externalTM() { + assertThat(this.config.input2Container.getContainerProperties().getTransactionManager()) + .isSameAs(this.config.tm); + Object handler = KafkaTestUtils.getPropertyValue(this.config.output2, "dispatcher.handlers", Set.class) + .iterator().next(); + assertThat(KafkaTestUtils.getPropertyValue(handler, "delegate.kafkaTemplate.producerFactory")) + .isSameAs(this.config.pf); + } + + @EnableBinding(TwoProcessors.class) + @EnableAutoConfiguration + public static class Config { + + final List outs = new ArrayList<>(); + + final CountDownLatch latch = new CountDownLatch(2); + + @Autowired + private MessageChannel output; + + @Autowired + MessageChannel output2; + + AbstractMessageListenerContainer input2Container; + + ProducerFactory pf; + + KafkaAwareTransactionManager tm; + + @KafkaListener(id = "test.cons.prod", topics = "consumer.producer.txOut") + public void listenOut(String in) { + this.outs.add(in); + this.latch.countDown(); + } + + @StreamListener(Processor.INPUT) + public void listenIn(String in) { + this.output.send(new GenericMessage<>(in.toUpperCase())); + if (in.equals("two")) { + throw new RuntimeException("fail"); + } + } + + @StreamListener("input2") + public void listenIn2(String in) { + } + + @Bean + public ApplicationRunner runner(KafkaTemplate template) { + return args -> { + template.send("consumer.producer.txIn", "one".getBytes()); + template.send("consumer.producer.txIn", "two".getBytes()); + template.send("consumer.producer.txIn", "three".getBytes()); + }; + } + + @Bean + public ListenerContainerCustomizer> customizer() { + return (container, dest, group) -> { + container.setAfterRollbackProcessor(new DefaultAfterRollbackProcessor<>(new FixedBackOff(0L, 1L))); + if ("input2".equals(dest)) { + this.input2Container = container; + } + }; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Bean + public KafkaAwareTransactionManager tm(ProducerFactory pf) { + KafkaAwareTransactionManager mock = mock(KafkaAwareTransactionManager.class); + this.pf = pf; + given(mock.getProducerFactory()).willReturn(pf); + this.tm = mock; + return mock; + } + + } + + public interface TwoProcessors extends Processor { + + @Input + SubscribableChannel input2(); + + @Output + MessageChannel output2(); + + } + +} diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/binder-config-autoconfig.properties b/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/binder-config-autoconfig.properties new file mode 100644 index 000000000..38d0d9f2a --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/binder-config-autoconfig.properties @@ -0,0 +1,10 @@ +spring.kafka.producer.keySerializer=org.apache.kafka.common.serialization.LongSerializer +spring.kafka.producer.valueSerializer=org.apache.kafka.common.serialization.LongSerializer +spring.kafka.consumer.keyDeserializer=org.apache.kafka.common.serialization.LongDeserializer +spring.kafka.consumer.valueDeserializer=org.apache.kafka.common.serialization.LongDeserializer +spring.kafka.producer.batchSize=10 +spring.kafka.bootstrapServers=10.98.09.199:9092,10.98.09.196:9092 +spring.kafka.producer.compressionType=snappy +# Test consumer properties +spring.kafka.consumer.auto-offset-reset=earliest +spring.kafka.consumer.group-id=groupIdFromBootConfig diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/binder-config.properties b/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/binder-config.properties new file mode 100644 index 000000000..5a1096324 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/binder-config.properties @@ -0,0 +1 @@ +spring.cloud.stream.kafka.binder.brokers=10.98.09.199:9082 diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/jaas-sample-kafka-only.conf b/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/jaas-sample-kafka-only.conf new file mode 100644 index 000000000..9b6891fb8 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/jaas-sample-kafka-only.conf @@ -0,0 +1,7 @@ +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + keyTab="/etc/security/keytabs/kafka_client.keytab" + principal="kafka-client-1@EXAMPLE.COM"; +}; \ No newline at end of file diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/jaas-sample-with-zk.conf b/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/jaas-sample-with-zk.conf new file mode 100644 index 000000000..fe511bd62 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/jaas-sample-with-zk.conf @@ -0,0 +1,14 @@ +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + keyTab="/etc/security/keytabs/kafka_client.keytab" + principal="kafka-client-1@EXAMPLE.COM"; +}; +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + keyTab="/etc/security/keytabs/zk_client.keytab" + principal="zk-client-1@EXAMPLE.COM"; +}; \ No newline at end of file diff --git a/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/logback.xml b/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/logback.xml new file mode 100644 index 000000000..77ab1d058 --- /dev/null +++ b/k-binder/spring-cloud-stream-binder-kafka/src/test/resources/logback.xml @@ -0,0 +1,15 @@ + + + + %d{ISO8601} %5p %t %c{2}:%L - %m%n + + + + + + + + + + + diff --git a/k-binder/update-version.sh b/k-binder/update-version.sh new file mode 100755 index 000000000..bc0672b8e --- /dev/null +++ b/k-binder/update-version.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +#Execute this script from local checkout of spring cloud stream + +./mvnw versions:update-parent -DparentVersion=[0.0.1,$2] -Pspring -DgenerateBackupPoms=false -DallowSnapshots=true +./mvnw versions:set -DnewVersion=$1 -DgenerateBackupPoms=false + + + +lines=$(find . -name 'pom.xml' | xargs egrep "SNAPSHOT|M[0-9]|RC[0-9]" | grep -v regex | wc -l) +if [ $lines -eq 0 ]; then + echo "No snapshots found" +else + echo "Snapshots found." +fi + +lines=$(find . -name 'pom.xml' | xargs egrep "M[0-9]" | grep -v regex | wc -l) +if [ $lines -eq 0 ]; then + echo "No milestones found" +else + echo "Milestones found." +fi + +lines=$(find . -name 'pom.xml' | xargs egrep "RC[0-9]" | grep -v regex | wc -l) +if [ $lines -eq 0 ]; then + echo "No release candidates found" +else + echo "Release candidates found." +fi