Initial commit
This commit is contained in:
Marcin Grzejszczak
@@ -1,18 +0,0 @@
|
||||
# EditorConfig is awesome: http://EditorConfig.org
|
||||
|
||||
# top-most EditorConfig file
|
||||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = tab
|
||||
indent_size = 4
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
|
||||
[*.yml]
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
|
||||
[*.java]
|
||||
ij_java_names_count_to_use_import_on_demand = 999
|
||||
ij_java_class_count_to_use_import_on_demand = 999
|
||||
37
.github/ISSUE_TEMPLATE/new-issue.md
vendored
37
.github/ISSUE_TEMPLATE/new-issue.md
vendored
@@ -1,37 +0,0 @@
|
||||
---
|
||||
name: New Issue
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
Thanks for considering raising a Spring Cloud Stream issue. Please take the time to review the following categories as some of them do not apply here.
|
||||
|
||||
🙅 "Please DO NOT Raise an Issue" Cases
|
||||
- Question
|
||||
STOP!! Please ask questions about how to use something, or to understand why something isn't working as you expect it to, on Stack Overflow using the spring-cloud-stream tag.
|
||||
- Security Vulnerability
|
||||
STOP!! Please don't raise security vulnerabilities here. Head over to https://pivotal.io/security to learn how to disclose them responsibly.
|
||||
- With an Immediate Pull Request
|
||||
An issue will be closed as a duplicate of the immediate pull request, so you don't have to raise an issue if you plan to create a pull request immediately.
|
||||
|
||||
If your case does not match any of he above categories. . .
|
||||
|
||||
**Describe the issue**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
1. Go to '...'
|
||||
2. Click on '....'
|
||||
3. Scroll down to '....'
|
||||
4. See error
|
||||
|
||||
**Version of the framework**
|
||||
**Expected behavior**
|
||||
**Screenshots**
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
||||
53
.github/workflows/deploy-docs.yml
vendored
Normal file
53
.github/workflows/deploy-docs.yml
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
name: Deploy Docs
|
||||
run-name: ${{ format('{0} ({1})', github.workflow, github.event.inputs.build-refname || 'all') }}
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build-refname:
|
||||
description: Enter git refname to build (e.g., 5.7.x).
|
||||
required: false
|
||||
push:
|
||||
branches: docs-build
|
||||
env:
|
||||
GRADLE_ENTERPRISE_SECRET_ACCESS_KEY: ${{ secrets.GRADLE_ENTERPRISE_SECRET_ACCESS_KEY }}
|
||||
permissions:
|
||||
contents: write
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository_owner == 'spring-cloud'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
- name: Set up refname build
|
||||
if: github.event.inputs.build-refname
|
||||
run: |
|
||||
git fetch --depth 1 https://github.com/$GITHUB_REPOSITORY ${{ github.event.inputs.build-refname }}
|
||||
export BUILD_REFNAME=${{ github.event.inputs.build-refname }}
|
||||
echo "BUILD_REFNAME=$BUILD_REFNAME" >> $GITHUB_ENV
|
||||
export BUILD_VERSION=$(git cat-file --textconv FETCH_HEAD:pom.xml | python3 -c "import xml.etree.ElementTree as xml; from sys import stdin; print(xml.parse(stdin).getroot().find('{http://maven.apache.org/POM/4.0.0}version').text)")
|
||||
echo BUILD_VERSION=$BUILD_VERSION >> $GITHUB_ENV
|
||||
- name: Run Antora
|
||||
run: |
|
||||
./mvnw --no-transfer-progress -B antora
|
||||
- name: Publish Docs
|
||||
uses: rwinch/spring-doc-actions/rsync-antora-reference@httpdocs-path
|
||||
with:
|
||||
docs-username: ${{ secrets.DOCS_USERNAME }}
|
||||
docs-host: ${{ secrets.DOCS_HOST }}
|
||||
docs-ssh-key: ${{ secrets.DOCS_SSH_KEY }}
|
||||
docs-ssh-host-key: ${{ secrets.DOCS_SSH_HOST_KEY }}
|
||||
site-path: target/antora/site
|
||||
- name: Bust Cloudflare Cache
|
||||
uses: spring-io/spring-doc-actions/bust-cloudflare-antora-cache@v0.0.11
|
||||
with:
|
||||
context-root: spring-cloud-stream
|
||||
cloudflare-zone-id: ${{ secrets.CLOUDFLARE_ZONE_ID }}
|
||||
cloudflare-cache-token: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }}
|
||||
35
.gitignore
vendored
35
.gitignore
vendored
@@ -1,27 +1,22 @@
|
||||
/application.yml
|
||||
/application.properties
|
||||
asciidoctor.css
|
||||
*~
|
||||
.#*
|
||||
*#
|
||||
target/
|
||||
build/
|
||||
bin/
|
||||
_site/
|
||||
.classpath
|
||||
.settings/
|
||||
.project
|
||||
.settings
|
||||
.classpath
|
||||
*.orig
|
||||
.springBeans
|
||||
.sts4-cache/
|
||||
.attach_pid*
|
||||
.factorypath
|
||||
.sts4-cache
|
||||
.ant-targets-build.xml
|
||||
src/ant/.ant-targets-upload-dist.xml
|
||||
*.sonar4clipse*
|
||||
.DS_Store
|
||||
*.sw*
|
||||
*.iml
|
||||
*.ipr
|
||||
*.iws
|
||||
.idea/*
|
||||
.factorypath
|
||||
dump.rdb
|
||||
.apt_generated
|
||||
artifacts
|
||||
**/dependency-reduced-pom.xml
|
||||
/.idea/
|
||||
*.graphml
|
||||
node
|
||||
node_modules
|
||||
build
|
||||
package.json
|
||||
package-lock.json
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
-DaltSnapshotDeploymentRepository=repo.spring.io::default::https://repo.spring.io/libs-snapshot-local -P spring
|
||||
@@ -1,68 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<settings>
|
||||
<servers>
|
||||
<server>
|
||||
<id>repo.spring.io</id>
|
||||
<username>${env.CI_DEPLOY_USERNAME}</username>
|
||||
<password>${env.CI_DEPLOY_PASSWORD}</password>
|
||||
</server>
|
||||
</servers>
|
||||
<profiles>
|
||||
<profile>
|
||||
<!--
|
||||
N.B. this profile is only here to support users and IDEs that do not use Maven 3.3.
|
||||
It isn't needed on the command line if you use the wrapper script (mvnw) or if you use
|
||||
a native Maven with the right version. Eclipse users should points their Maven tooling to
|
||||
this settings file, or copy the profile into their ~/.m2/settings.xml.
|
||||
-->
|
||||
<id>spring</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>spring-snapshots</id>
|
||||
<name>Spring Snapshots</name>
|
||||
<url>https://repo.spring.io/libs-snapshot-local</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
<repository>
|
||||
<id>spring-milestones</id>
|
||||
<name>Spring Milestones</name>
|
||||
<url>https://repo.spring.io/libs-milestone-local</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
<repository>
|
||||
<id>spring-releases</id>
|
||||
<name>Spring Releases</name>
|
||||
<url>https://repo.spring.io/release</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
</repositories>
|
||||
<pluginRepositories>
|
||||
<pluginRepository>
|
||||
<id>spring-snapshots</id>
|
||||
<name>Spring Snapshots</name>
|
||||
<url>https://repo.spring.io/libs-snapshot-local</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
</pluginRepository>
|
||||
<pluginRepository>
|
||||
<id>spring-milestones</id>
|
||||
<name>Spring Milestones</name>
|
||||
<url>https://repo.spring.io/libs-milestone-local</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
</profile>
|
||||
</profiles>
|
||||
</settings>
|
||||
17
.travis.yml
17
.travis.yml
@@ -1,17 +0,0 @@
|
||||
sudo: required
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.m2
|
||||
language: java
|
||||
jdk:
|
||||
- oraclejdk8
|
||||
install: true
|
||||
# The environment variable ${TRAVIS_PULL_REQUEST} is set to "false" when the build
|
||||
# is for a normal branch commit. When the build is for a pull request, it will
|
||||
# contain the pull request’s number.
|
||||
script:
|
||||
- '[ "${TRAVIS_PULL_REQUEST}" != "false" ] || ./mvnw package -Pspring,full -U -Dmaven.test.redirectTestOutputToFile=false'
|
||||
- '[ "${TRAVIS_PULL_REQUEST}" = "false" ] || ./mvnw package -Pspring -U -Dmaven.test.redirectTestOutputToFile=false'
|
||||
after_success:
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
dist: trusty
|
||||
@@ -1,44 +0,0 @@
|
||||
= Contributor Code of Conduct
|
||||
|
||||
As contributors and maintainers of this project, and in the interest of fostering an open
|
||||
and welcoming community, we pledge to respect all people who contribute through reporting
|
||||
issues, posting feature requests, updating documentation, submitting pull requests or
|
||||
patches, and other activities.
|
||||
|
||||
We are committed to making participation in this project a harassment-free experience for
|
||||
everyone, regardless of level of experience, gender, gender identity and expression,
|
||||
sexual orientation, disability, personal appearance, body size, race, ethnicity, age,
|
||||
religion, or nationality.
|
||||
|
||||
Examples of unacceptable behavior by participants include:
|
||||
|
||||
* The use of sexualized language or imagery
|
||||
* Personal attacks
|
||||
* Trolling or insulting/derogatory comments
|
||||
* Public or private harassment
|
||||
* Publishing other's private information, such as physical or electronic addresses,
|
||||
without explicit permission
|
||||
* Other unethical or unprofessional conduct
|
||||
|
||||
Project maintainers have the right and responsibility to remove, edit, or reject comments,
|
||||
commits, code, wiki edits, issues, and other contributions that are not aligned to this
|
||||
Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors
|
||||
that they deem inappropriate, threatening, offensive, or harmful.
|
||||
|
||||
By adopting this Code of Conduct, project maintainers commit themselves to fairly and
|
||||
consistently applying these principles to every aspect of managing this project. Project
|
||||
maintainers who do not follow or enforce the Code of Conduct may be permanently removed
|
||||
from the project team.
|
||||
|
||||
This Code of Conduct applies both within project spaces and in public spaces when an
|
||||
individual is representing the project or its community.
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by
|
||||
contacting a project maintainer at spring-code-of-conduct@pivotal.io . All complaints will
|
||||
be reviewed and investigated and will result in a response that is deemed necessary and
|
||||
appropriate to the circumstances. Maintainers are obligated to maintain confidentiality
|
||||
with regard to the reporter of an incident.
|
||||
|
||||
This Code of Conduct is adapted from the
|
||||
http://contributor-covenant.org[Contributor Covenant], version 1.3.0, available at
|
||||
http://contributor-covenant.org/version/1/3/0/[contributor-covenant.org/version/1/3/0/]
|
||||
202
LICENSE
202
LICENSE
@@ -1,202 +0,0 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright {yyyy} {name of copyright owner}
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
410
README.adoc
410
README.adoc
@@ -1,409 +1,23 @@
|
||||
////
|
||||
DO NOT EDIT THIS FILE. IT WAS GENERATED.
|
||||
Manual changes to this file will be lost when it is generated again.
|
||||
Edit the files in the src/main/asciidoc/ directory instead.
|
||||
////
|
||||
= Spring Cloud Stream Docs Build
|
||||
|
||||
You're currently viewing the Antora playbook branch.
|
||||
The playbook branch hosts the docs build that is used to build and publish the production docs site.
|
||||
|
||||
:jdkversion: 17
|
||||
:github-tag: master
|
||||
:github-repo: spring-cloud/spring-cloud-stream
|
||||
The Spring Cloud Stream reference docs are built using https://antora.org[Antora].
|
||||
This README covers how to build the docs in a software branch as well as how to build the production docs site locally.
|
||||
|
||||
:github-raw: https://raw.githubusercontent.com/{github-repo}/{github-tag}
|
||||
:github-code: https://github.com/{github-repo}/tree/{github-tag}
|
||||
== Building the Site
|
||||
|
||||
image::https://circleci.com/gh/spring-cloud/spring-cloud-stream.svg?style=svg["CircleCI", link="https://circleci.com/gh/spring-cloud/spring-cloud-stream"]
|
||||
image::https://codecov.io/gh/spring-cloud/spring-cloud-stream/branch/{github-tag}/graph/badge.svg["codecov", link="https://codecov.io/gh/spring-cloud/spring-cloud-stream"]
|
||||
image::https://badges.gitter.im/spring-cloud/spring-cloud-stream.svg[Gitter, link="https://gitter.im/spring-cloud/spring-cloud-stream?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge"]
|
||||
You can build the entire site by invoking the following on the docs-build branch and then viewing the site at `target/site/index.html`
|
||||
|
||||
// ======================================================================================
|
||||
|
||||
== Introduction
|
||||
Spring Cloud Stream is a framework for building message-driven microservice applications.
|
||||
Spring Cloud Stream builds upon Spring Boot to create standalone, production-grade Spring applications and uses Spring Integration to provide connectivity to message brokers.
|
||||
It provides opinionated configuration of middleware from several vendors, introducing the concepts of persistent publish-subscribe semantics, consumer groups, and partitions.
|
||||
These are called binder implementations in the parlance of Spring Cloud Stream.
|
||||
Out of the box, Spring Cloud Stream provides binder implementations for Apache Kafka and RabbitMQ.
|
||||
While these two binder implementations are based on Message Channels, Spring Cloud Stream also provides another binder implementation for Kafka Streams that does not use message channels, but native Kafka Streams types such as KStream, KTable etc.
|
||||
|
||||
Below, you can find more information on how to use these various out-of-the-box binder implementations in your applications.
|
||||
|
||||
== Apache Kafka Binder
|
||||
|
||||
=== Usage
|
||||
|
||||
To use Apache Kafka binder, you need to add `spring-cloud-stream-binder-kafka` as a dependency to your Spring Cloud Stream application, as shown in the following example for Maven:
|
||||
|
||||
[source,xml]
|
||||
[source,bash]
|
||||
----
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-binder-kafka</artifactId>
|
||||
</dependency>
|
||||
./mvnw antora
|
||||
----
|
||||
|
||||
Alternatively, you can also use the Spring Cloud Stream Kafka Starter, as shown in the following example for Maven:
|
||||
== Building a Specific Branch
|
||||
|
||||
[source,xml]
|
||||
[source,bash]
|
||||
----
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-stream-kafka</artifactId>
|
||||
</dependency>
|
||||
./mvnw antora
|
||||
----
|
||||
|
||||
== Apache Kafka Streams Binder
|
||||
|
||||
=== Usage
|
||||
|
||||
To use Apache Kafka Streams binder, you need to add `spring-cloud-stream-binder-kafka-streams` as a dependency to your Spring Cloud Stream application, as shown in the following example for Maven:
|
||||
|
||||
[source,xml]
|
||||
----
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-binder-kafka-streams</artifactId>
|
||||
</dependency>
|
||||
----
|
||||
|
||||
== RabbitMQ Binder
|
||||
=== Usage
|
||||
|
||||
To use the RabbitMQ binder, you can add it to your Spring Cloud Stream application, by using the following Maven coordinates:
|
||||
|
||||
[source,xml]
|
||||
----
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-binder-rabbit</artifactId>
|
||||
</dependency>
|
||||
----
|
||||
|
||||
Alternatively, you can use the Spring Cloud Stream RabbitMQ Starter, as follows:
|
||||
|
||||
[source,xml]
|
||||
----
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-stream-rabbit</artifactId>
|
||||
</dependency>
|
||||
----
|
||||
|
||||
== Resources
|
||||
|
||||
For more information, please visit the https://spring.io/projects/spring-cloud-stream[project website]:
|
||||
|
||||
== Building
|
||||
|
||||
:jdkversion: 17
|
||||
|
||||
=== Basic Compile and Test
|
||||
|
||||
To build the source you will need to install JDK {jdkversion}.
|
||||
|
||||
Spring Cloud uses Maven for most build-related activities, and you
|
||||
should be able to get off the ground quite quickly by cloning the
|
||||
project you are interested in and typing
|
||||
|
||||
----
|
||||
$ ./mvnw install
|
||||
----
|
||||
|
||||
NOTE: You can also install Maven (>=3.3.3) yourself and run the `mvn` command
|
||||
in place of `./mvnw` in the examples below. If you do that you also
|
||||
might need to add `-P spring` if your local Maven settings do not
|
||||
contain repository declarations for spring pre-release artifacts.
|
||||
|
||||
NOTE: Be aware that you might need to increase the amount of memory
|
||||
available to Maven by setting a `MAVEN_OPTS` environment variable with
|
||||
a value like `-Xmx512m -XX:MaxPermSize=128m`. We try to cover this in
|
||||
the `.mvn` configuration, so if you find you have to do it to make a
|
||||
build succeed, please raise a ticket to get the settings added to
|
||||
source control.
|
||||
|
||||
The projects that require middleware (i.e. Redis) for testing generally
|
||||
require that a local instance of [Docker](https://www.docker.com/get-started) is installed and running.
|
||||
|
||||
|
||||
=== Documentation
|
||||
|
||||
The spring-cloud-build module has a "docs" profile, and if you switch
|
||||
that on it will try to build asciidoc sources from
|
||||
`src/main/asciidoc`. As part of that process it will look for a
|
||||
`README.adoc` and process it by loading all the includes, but not
|
||||
parsing or rendering it, just copying it to `${main.basedir}`
|
||||
(defaults to `${basedir}`, i.e. the root of the project). If there are
|
||||
any changes in the README it will then show up after a Maven build as
|
||||
a modified file in the correct place. Just commit it and push the change.
|
||||
|
||||
=== Working with the code
|
||||
If you don't have an IDE preference we would recommend that you use
|
||||
https://www.springsource.com/developer/sts[Spring Tools Suite] or
|
||||
https://eclipse.org[Eclipse] when working with the code. We use the
|
||||
https://eclipse.org/m2e/[m2eclipse] eclipse plugin for maven support. Other IDEs and tools
|
||||
should also work without issue as long as they use Maven 3.3.3 or better.
|
||||
|
||||
==== Activate the Spring Maven profile
|
||||
Spring Cloud projects require the 'spring' Maven profile to be activated to resolve
|
||||
the spring milestone and snapshot repositories. Use your preferred IDE to set this
|
||||
profile to be active, or you may experience build errors.
|
||||
|
||||
==== Importing into eclipse with m2eclipse
|
||||
We recommend the https://eclipse.org/m2e/[m2eclipse] eclipse plugin when working with
|
||||
eclipse. If you don't already have m2eclipse installed it is available from the "eclipse
|
||||
marketplace".
|
||||
|
||||
NOTE: Older versions of m2e do not support Maven 3.3, so once the
|
||||
projects are imported into Eclipse you will also need to tell
|
||||
m2eclipse to use the right profile for the projects. If you
|
||||
see many different errors related to the POMs in the projects, check
|
||||
that you have an up to date installation. If you can't upgrade m2e,
|
||||
add the "spring" profile to your `settings.xml`. Alternatively you can
|
||||
copy the repository settings from the "spring" profile of the parent
|
||||
pom into your `settings.xml`.
|
||||
|
||||
==== Importing into eclipse without m2eclipse
|
||||
If you prefer not to use m2eclipse you can generate eclipse project metadata using the
|
||||
following command:
|
||||
|
||||
[indent=0]
|
||||
----
|
||||
$ ./mvnw eclipse:eclipse
|
||||
----
|
||||
|
||||
The generated eclipse projects can be imported by selecting `import existing projects`
|
||||
from the `file` menu.
|
||||
|
||||
|
||||
== Contributing
|
||||
|
||||
:spring-cloud-build-branch: master
|
||||
|
||||
Spring Cloud is released under the non-restrictive Apache 2.0 license,
|
||||
and follows a very standard Github development process, using Github
|
||||
tracker for issues and merging pull requests into master. If you want
|
||||
to contribute even something trivial please do not hesitate, but
|
||||
follow the guidelines below.
|
||||
|
||||
=== Sign the Contributor License Agreement
|
||||
Before we accept a non-trivial patch or pull request we will need you to sign the
|
||||
https://cla.pivotal.io/sign/spring[Contributor License Agreement].
|
||||
Signing the contributor's agreement does not grant anyone commit rights to the main
|
||||
repository, but it does mean that we can accept your contributions, and you will get an
|
||||
author credit if we do. Active contributors might be asked to join the core team, and
|
||||
given the ability to merge pull requests.
|
||||
|
||||
=== Code of Conduct
|
||||
This project adheres to the Contributor Covenant https://github.com/spring-cloud/spring-cloud-build/blob/master/docs/src/main/asciidoc/code-of-conduct.adoc[code of
|
||||
conduct]. By participating, you are expected to uphold this code. Please report
|
||||
unacceptable behavior to spring-code-of-conduct@pivotal.io.
|
||||
|
||||
=== Code Conventions and Housekeeping
|
||||
None of these is essential for a pull request, but they will all help. They can also be
|
||||
added after the original pull request but before a merge.
|
||||
|
||||
* Use the Spring Framework code format conventions. If you use Eclipse
|
||||
you can import formatter settings using the
|
||||
`eclipse-code-formatter.xml` file from the
|
||||
https://raw.githubusercontent.com/spring-cloud/spring-cloud-build/master/spring-cloud-dependencies-parent/eclipse-code-formatter.xml[Spring
|
||||
Cloud Build] project. If using IntelliJ, you can use the
|
||||
https://plugins.jetbrains.com/plugin/6546[Eclipse Code Formatter
|
||||
Plugin] to import the same file.
|
||||
* Make sure all new `.java` files to have a simple Javadoc class comment with at least an
|
||||
`@author` tag identifying you, and preferably at least a paragraph on what the class is
|
||||
for.
|
||||
* Add the ASF license header comment to all new `.java` files (copy from existing files
|
||||
in the project)
|
||||
* Add yourself as an `@author` to the .java files that you modify substantially (more
|
||||
than cosmetic changes).
|
||||
* Add some Javadocs and, if you change the namespace, some XSD doc elements.
|
||||
* A few unit tests would help a lot as well -- someone has to do it.
|
||||
* If no-one else is using your branch, please rebase it against the current master (or
|
||||
other target branch in the main project).
|
||||
* When writing a commit message please follow https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html[these conventions],
|
||||
if you are fixing an existing issue please add `Fixes gh-XXXX` at the end of the commit
|
||||
message (where XXXX is the issue number).
|
||||
|
||||
=== Checkstyle
|
||||
|
||||
Spring Cloud Build comes with a set of checkstyle rules. You can find them in the `spring-cloud-build-tools` module. The most notable files under the module are:
|
||||
|
||||
.spring-cloud-build-tools/
|
||||
----
|
||||
└── src
|
||||
├── checkstyle
|
||||
│ └── checkstyle-suppressions.xml <3>
|
||||
└── main
|
||||
└── resources
|
||||
├── checkstyle-header.txt <2>
|
||||
└── checkstyle.xml <1>
|
||||
----
|
||||
<1> Default Checkstyle rules
|
||||
<2> File header setup
|
||||
<3> Default suppression rules
|
||||
|
||||
==== Checkstyle configuration
|
||||
|
||||
Checkstyle rules are *disabled by default*. To add checkstyle to your project just define the following properties and plugins.
|
||||
|
||||
.pom.xml
|
||||
----
|
||||
<properties>
|
||||
<maven-checkstyle-plugin.failsOnError>true</maven-checkstyle-plugin.failsOnError> <1>
|
||||
<maven-checkstyle-plugin.failsOnViolation>true
|
||||
</maven-checkstyle-plugin.failsOnViolation> <2>
|
||||
<maven-checkstyle-plugin.includeTestSourceDirectory>true
|
||||
</maven-checkstyle-plugin.includeTestSourceDirectory> <3>
|
||||
</properties>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin> <4>
|
||||
<groupId>io.spring.javaformat</groupId>
|
||||
<artifactId>spring-javaformat-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin> <5>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
<reporting>
|
||||
<plugins>
|
||||
<plugin> <5>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</reporting>
|
||||
</build>
|
||||
----
|
||||
<1> Fails the build upon Checkstyle errors
|
||||
<2> Fails the build upon Checkstyle violations
|
||||
<3> Checkstyle analyzes also the test sources
|
||||
<4> Add the Spring Java Format plugin that will reformat your code to pass most of the Checkstyle formatting rules
|
||||
<5> Add checkstyle plugin to your build and reporting phases
|
||||
|
||||
If you need to suppress some rules (e.g. line length needs to be longer), then it's enough for you to define a file under `${project.root}/src/checkstyle/checkstyle-suppressions.xml` with your suppressions. Example:
|
||||
|
||||
.projectRoot/src/checkstyle/checkstyle-suppresions.xml
|
||||
----
|
||||
<?xml version="1.0"?>
|
||||
<!DOCTYPE suppressions PUBLIC
|
||||
"-//Puppy Crawl//DTD Suppressions 1.1//EN"
|
||||
"https://www.puppycrawl.com/dtds/suppressions_1_1.dtd">
|
||||
<suppressions>
|
||||
<suppress files=".*ConfigServerApplication\.java" checks="HideUtilityClassConstructor"/>
|
||||
<suppress files=".*ConfigClientWatch\.java" checks="LineLengthCheck"/>
|
||||
</suppressions>
|
||||
----
|
||||
|
||||
It's advisable to copy the `${spring-cloud-build.rootFolder}/.editorconfig` and `${spring-cloud-build.rootFolder}/.springformat` to your project. That way, some default formatting rules will be applied. You can do so by running this script:
|
||||
|
||||
```bash
|
||||
$ curl https://raw.githubusercontent.com/spring-cloud/spring-cloud-build/master/.editorconfig -o .editorconfig
|
||||
$ touch .springformat
|
||||
```
|
||||
|
||||
=== IDE setup
|
||||
|
||||
==== Intellij IDEA
|
||||
|
||||
In order to setup Intellij you should import our coding conventions, inspection profiles and set up the checkstyle plugin.
|
||||
The following files can be found in the https://github.com/spring-cloud/spring-cloud-build/tree/master/spring-cloud-build-tools[Spring Cloud Build] project.
|
||||
|
||||
.spring-cloud-build-tools/
|
||||
----
|
||||
└── src
|
||||
├── checkstyle
|
||||
│ └── checkstyle-suppressions.xml <3>
|
||||
└── main
|
||||
└── resources
|
||||
├── checkstyle-header.txt <2>
|
||||
├── checkstyle.xml <1>
|
||||
└── intellij
|
||||
├── Intellij_Project_Defaults.xml <4>
|
||||
└── Intellij_Spring_Boot_Java_Conventions.xml <5>
|
||||
----
|
||||
<1> Default Checkstyle rules
|
||||
<2> File header setup
|
||||
<3> Default suppression rules
|
||||
<4> Project defaults for Intellij that apply most of Checkstyle rules
|
||||
<5> Project style conventions for Intellij that apply most of Checkstyle rules
|
||||
|
||||
.Code style
|
||||
|
||||
image::https://raw.githubusercontent.com/spring-cloud/spring-cloud-build/{spring-cloud-build-branch}/docs/src/main/asciidoc/images/intellij-code-style.png[Code style]
|
||||
|
||||
Go to `File` -> `Settings` -> `Editor` -> `Code style`. There click on the icon next to the `Scheme` section. There, click on the `Import Scheme` value and pick the `Intellij IDEA code style XML` option. Import the `spring-cloud-build-tools/src/main/resources/intellij/Intellij_Spring_Boot_Java_Conventions.xml` file.
|
||||
|
||||
.Inspection profiles
|
||||
|
||||
image::https://raw.githubusercontent.com/spring-cloud/spring-cloud-build/{spring-cloud-build-branch}/docs/src/main/asciidoc/images/intellij-inspections.png[Code style]
|
||||
|
||||
Go to `File` -> `Settings` -> `Editor` -> `Inspections`. There click on the icon next to the `Profile` section. There, click on the `Import Profile` and import the `spring-cloud-build-tools/src/main/resources/intellij/Intellij_Project_Defaults.xml` file.
|
||||
|
||||
.Checkstyle
|
||||
|
||||
To have Intellij work with Checkstyle, you have to install the `Checkstyle` plugin. It's advisable to also install the `Assertions2Assertj` to automatically convert the JUnit assertions
|
||||
|
||||
image::https://raw.githubusercontent.com/spring-cloud/spring-cloud-build/{spring-cloud-build-branch}/docs/src/main/asciidoc/images/intellij-checkstyle.png[Checkstyle]
|
||||
|
||||
Go to `File` -> `Settings` -> `Other settings` -> `Checkstyle`. There click on the `+` icon in the `Configuration file` section. There, you'll have to define where the checkstyle rules should be picked from. In the image above, we've picked the rules from the cloned Spring Cloud Build repository. However, you can point to the Spring Cloud Build's GitHub repository (e.g. for the `checkstyle.xml` : `https://raw.githubusercontent.com/spring-cloud/spring-cloud-build/master/spring-cloud-build-tools/src/main/resources/checkstyle.xml`). We need to provide the following variables:
|
||||
|
||||
- `checkstyle.header.file` - please point it to the Spring Cloud Build's, `spring-cloud-build-tools/src/main/resources/checkstyle-header.txt` file either in your cloned repo or via the `https://raw.githubusercontent.com/spring-cloud/spring-cloud-build/master/spring-cloud-build-tools/src/main/resources/checkstyle-header.txt` URL.
|
||||
- `checkstyle.suppressions.file` - default suppressions. Please point it to the Spring Cloud Build's, `spring-cloud-build-tools/src/checkstyle/checkstyle-suppressions.xml` file either in your cloned repo or via the `https://raw.githubusercontent.com/spring-cloud/spring-cloud-build/master/spring-cloud-build-tools/src/checkstyle/checkstyle-suppressions.xml` URL.
|
||||
- `checkstyle.additional.suppressions.file` - this variable corresponds to suppressions in your local project. E.g. you're working on `spring-cloud-contract`. Then point to the `project-root/src/checkstyle/checkstyle-suppressions.xml` folder. Example for `spring-cloud-contract` would be: `/home/username/spring-cloud-contract/src/checkstyle/checkstyle-suppressions.xml`.
|
||||
|
||||
IMPORTANT: Remember to set the `Scan Scope` to `All sources` since we apply checkstyle rules for production and test sources.
|
||||
|
||||
=== Duplicate Finder
|
||||
|
||||
Spring Cloud Build brings along the `basepom:duplicate-finder-maven-plugin`, that enables flagging duplicate and conflicting classes and resources on the java classpath.
|
||||
|
||||
==== Duplicate Finder configuration
|
||||
|
||||
Duplicate finder is *enabled by default* and will run in the `verify` phase of your Maven build, but it will only take effect in your project if you add the `duplicate-finder-maven-plugin` to the `build` section of the projecst's `pom.xml`.
|
||||
|
||||
.pom.xml
|
||||
[source,xml]
|
||||
----
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.basepom.maven</groupId>
|
||||
<artifactId>duplicate-finder-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
----
|
||||
|
||||
For other properties, we have set defaults as listed in the https://github.com/basepom/duplicate-finder-maven-plugin/wiki[plugin documentation].
|
||||
|
||||
You can easily override them but setting the value of the selected property prefixed with `duplicate-finder-maven-plugin`. For example, set `duplicate-finder-maven-plugin.skip` to `true` in order to skip duplicates check in your build.
|
||||
|
||||
If you need to add `ignoredClassPatterns` or `ignoredResourcePatterns` to your setup, make sure to add them in the plugin configuration section of your project:
|
||||
|
||||
[source,xml]
|
||||
----
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.basepom.maven</groupId>
|
||||
<artifactId>duplicate-finder-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<ignoredClassPatterns>
|
||||
<ignoredClassPattern>org.joda.time.base.BaseDateTime</ignoredClassPattern>
|
||||
<ignoredClassPattern>.*module-info</ignoredClassPattern>
|
||||
</ignoredClassPatterns>
|
||||
<ignoredResourcePatterns>
|
||||
<ignoredResourcePattern>changelog.txt</ignoredResourcePattern>
|
||||
</ignoredResourcePatterns>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
----
|
||||
|
||||
|
||||
@@ -6,38 +6,39 @@ antora:
|
||||
- '@antora/collector-extension'
|
||||
- '@antora/atlas-extension'
|
||||
- require: '@springio/antora-extensions/root-component-extension'
|
||||
root_component_name: 'PROJECT_WITHOUT_SPRING'
|
||||
# FIXME: Run antora once using this extension to migrate to the Asciidoc Tabs syntax
|
||||
# and then remove this extension
|
||||
- require: '@springio/antora-extensions/tabs-migration-extension'
|
||||
unwrap_example_block: always
|
||||
save_result: true
|
||||
root_component_name: 'cloud-stream'
|
||||
site:
|
||||
title: PROJECT_FULL_NAME
|
||||
url: https://docs.spring.io/PROJECT_NAME/reference/
|
||||
title: Spring Cloud Stream
|
||||
url: https://docs.spring.io/spring-cloud-stream/reference
|
||||
robots: allow
|
||||
git:
|
||||
ensure_git_suffix: false
|
||||
content:
|
||||
sources:
|
||||
- url: ./..
|
||||
branches: HEAD
|
||||
- url: https://github.com/spring-cloud/spring-cloud-stream
|
||||
# Refname matching:
|
||||
# https://docs.antora.org/antora/latest/playbook/content-refname-matching/
|
||||
branches: [ main ]
|
||||
tags: [ '({4..9}).+({1..9}).+({0..9})?(-{RC,M}+({0..9}))', '!4.1.0-M1' ]
|
||||
start_path: docs
|
||||
worktrees: true
|
||||
asciidoc:
|
||||
attributes:
|
||||
page-stackoverflow-url: https://stackoverflow.com/tags/spring-cloud
|
||||
page-pagination: ''
|
||||
hide-uri-scheme: '@'
|
||||
tabs-sync-option: '@'
|
||||
chomp: 'all'
|
||||
extensions:
|
||||
- '@asciidoctor/tabs'
|
||||
- '@springio/asciidoctor-extensions'
|
||||
sourcemap: true
|
||||
urls:
|
||||
latest_version_segment_strategy: redirect:to
|
||||
latest_version_segment: ''
|
||||
redirect_facility: httpd
|
||||
ui:
|
||||
bundle:
|
||||
url: https://github.com/spring-io/antora-ui-spring/releases/download/v0.3.5/ui-bundle.zip
|
||||
snapshot: true
|
||||
runtime:
|
||||
log:
|
||||
failure_level: warn
|
||||
format: pretty
|
||||
ui:
|
||||
bundle:
|
||||
url: https://github.com/spring-io/antora-ui-spring/releases/download/v0.3.5/ui-bundle.zip
|
||||
26
binders/kafka-binder/.gitignore
vendored
26
binders/kafka-binder/.gitignore
vendored
@@ -1,26 +0,0 @@
|
||||
/application.yml
|
||||
/application.properties
|
||||
asciidoctor.css
|
||||
*~
|
||||
.#*
|
||||
*#
|
||||
target/
|
||||
build/
|
||||
bin/
|
||||
_site/
|
||||
.classpath
|
||||
.project
|
||||
.settings
|
||||
.springBeans
|
||||
.DS_Store
|
||||
*.sw*
|
||||
*.iml
|
||||
*.ipr
|
||||
*.iws
|
||||
.idea/*
|
||||
*/.idea
|
||||
.factorypath
|
||||
dump.rdb
|
||||
.apt_generated
|
||||
artifacts
|
||||
.sts4-cache
|
||||
@@ -1 +0,0 @@
|
||||
-Xmx1024m -XX:CICompilerCount=1 -XX:TieredStopAtLevel=1 -Djava.security.egd=file:/dev/./urandom
|
||||
@@ -1 +0,0 @@
|
||||
-DaltSnapshotDeploymentRepository=repo.spring.io::default::https://repo.spring.io/libs-snapshot-local -P spring
|
||||
@@ -1,117 +0,0 @@
|
||||
/*
|
||||
* Copyright 2007-present the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import java.net.*;
|
||||
import java.io.*;
|
||||
import java.nio.channels.*;
|
||||
import java.util.Properties;
|
||||
|
||||
public class MavenWrapperDownloader {
|
||||
|
||||
private static final String WRAPPER_VERSION = "0.5.6";
|
||||
/**
|
||||
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
|
||||
*/
|
||||
private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
|
||||
+ WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
|
||||
|
||||
/**
|
||||
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
|
||||
* use instead of the default one.
|
||||
*/
|
||||
private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
|
||||
".mvn/wrapper/maven-wrapper.properties";
|
||||
|
||||
/**
|
||||
* Path where the maven-wrapper.jar will be saved to.
|
||||
*/
|
||||
private static final String MAVEN_WRAPPER_JAR_PATH =
|
||||
".mvn/wrapper/maven-wrapper.jar";
|
||||
|
||||
/**
|
||||
* Name of the property which should be used to override the default download url for the wrapper.
|
||||
*/
|
||||
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
|
||||
|
||||
public static void main(String args[]) {
|
||||
System.out.println("- Downloader started");
|
||||
File baseDirectory = new File(args[0]);
|
||||
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
|
||||
|
||||
// If the maven-wrapper.properties exists, read it and check if it contains a custom
|
||||
// wrapperUrl parameter.
|
||||
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
|
||||
String url = DEFAULT_DOWNLOAD_URL;
|
||||
if(mavenWrapperPropertyFile.exists()) {
|
||||
FileInputStream mavenWrapperPropertyFileInputStream = null;
|
||||
try {
|
||||
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
|
||||
Properties mavenWrapperProperties = new Properties();
|
||||
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
|
||||
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
|
||||
} catch (IOException e) {
|
||||
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
|
||||
} finally {
|
||||
try {
|
||||
if(mavenWrapperPropertyFileInputStream != null) {
|
||||
mavenWrapperPropertyFileInputStream.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
// Ignore ...
|
||||
}
|
||||
}
|
||||
}
|
||||
System.out.println("- Downloading from: " + url);
|
||||
|
||||
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
|
||||
if(!outputFile.getParentFile().exists()) {
|
||||
if(!outputFile.getParentFile().mkdirs()) {
|
||||
System.out.println(
|
||||
"- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
|
||||
}
|
||||
}
|
||||
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
|
||||
try {
|
||||
downloadFileFromURL(url, outputFile);
|
||||
System.out.println("Done");
|
||||
System.exit(0);
|
||||
} catch (Throwable e) {
|
||||
System.out.println("- Error downloading");
|
||||
e.printStackTrace();
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
private static void downloadFileFromURL(String urlString, File destination) throws Exception {
|
||||
if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
|
||||
String username = System.getenv("MVNW_USERNAME");
|
||||
char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
|
||||
Authenticator.setDefault(new Authenticator() {
|
||||
@Override
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(username, password);
|
||||
}
|
||||
});
|
||||
}
|
||||
URL website = new URL(urlString);
|
||||
ReadableByteChannel rbc;
|
||||
rbc = Channels.newChannel(website.openStream());
|
||||
FileOutputStream fos = new FileOutputStream(destination);
|
||||
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
|
||||
fos.close();
|
||||
rbc.close();
|
||||
}
|
||||
|
||||
}
|
||||
BIN
binders/kafka-binder/.mvn/wrapper/maven-wrapper.jar
vendored
BIN
binders/kafka-binder/.mvn/wrapper/maven-wrapper.jar
vendored
Binary file not shown.
@@ -1,2 +0,0 @@
|
||||
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip
|
||||
wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar
|
||||
@@ -1,66 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<settings>
|
||||
<servers>
|
||||
<server>
|
||||
<id>repo.spring.io</id>
|
||||
<username>${env.CI_DEPLOY_USERNAME}</username>
|
||||
<password>${env.CI_DEPLOY_PASSWORD}</password>
|
||||
</server>
|
||||
</servers>
|
||||
<profiles>
|
||||
<profile>
|
||||
<!--
|
||||
N.B. this profile is only here to support users and IDEs that do not use Maven 3.3.
|
||||
It isn't needed on the command line if you use the wrapper script (mvnw) or if you use
|
||||
a native Maven with the right version. Eclipse users should points their Maven tooling to
|
||||
this settings file, or copy the profile into their ~/.m2/settings.xml.
|
||||
-->
|
||||
<id>spring</id>
|
||||
<activation><activeByDefault>true</activeByDefault></activation>
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>spring-snapshots</id>
|
||||
<name>Spring Snapshots</name>
|
||||
<url>https://repo.spring.io/libs-snapshot-local</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
<repository>
|
||||
<id>spring-milestones</id>
|
||||
<name>Spring Milestones</name>
|
||||
<url>https://repo.spring.io/libs-milestone-local</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
<repository>
|
||||
<id>spring-releases</id>
|
||||
<name>Spring Releases</name>
|
||||
<url>https://repo.spring.io/release</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
</repositories>
|
||||
<pluginRepositories>
|
||||
<pluginRepository>
|
||||
<id>spring-snapshots</id>
|
||||
<name>Spring Snapshots</name>
|
||||
<url>https://repo.spring.io/libs-snapshot-local</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
</pluginRepository>
|
||||
<pluginRepository>
|
||||
<id>spring-milestones</id>
|
||||
<name>Spring Milestones</name>
|
||||
<url>https://repo.spring.io/libs-milestone-local</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
</profile>
|
||||
</profiles>
|
||||
</settings>
|
||||
@@ -1,201 +0,0 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
https://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright {yyyy} {name of copyright owner}
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
310
binders/kafka-binder/mvnw
vendored
310
binders/kafka-binder/mvnw
vendored
@@ -1,310 +0,0 @@
|
||||
#!/bin/sh
|
||||
# ----------------------------------------------------------------------------
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Maven Start Up Batch script
|
||||
#
|
||||
# Required ENV vars:
|
||||
# ------------------
|
||||
# JAVA_HOME - location of a JDK home dir
|
||||
#
|
||||
# Optional ENV vars
|
||||
# -----------------
|
||||
# M2_HOME - location of maven2's installed home dir
|
||||
# MAVEN_OPTS - parameters passed to the Java VM when running Maven
|
||||
# e.g. to debug Maven itself, use
|
||||
# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
|
||||
# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
if [ -z "$MAVEN_SKIP_RC" ] ; then
|
||||
|
||||
if [ -f /etc/mavenrc ] ; then
|
||||
. /etc/mavenrc
|
||||
fi
|
||||
|
||||
if [ -f "$HOME/.mavenrc" ] ; then
|
||||
. "$HOME/.mavenrc"
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
# OS specific support. $var _must_ be set to either true or false.
|
||||
cygwin=false;
|
||||
darwin=false;
|
||||
mingw=false
|
||||
case "`uname`" in
|
||||
CYGWIN*) cygwin=true ;;
|
||||
MINGW*) mingw=true;;
|
||||
Darwin*) darwin=true
|
||||
# Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
|
||||
# See https://developer.apple.com/library/mac/qa/qa1170/_index.html
|
||||
if [ -z "$JAVA_HOME" ]; then
|
||||
if [ -x "/usr/libexec/java_home" ]; then
|
||||
export JAVA_HOME="`/usr/libexec/java_home`"
|
||||
else
|
||||
export JAVA_HOME="/Library/Java/Home"
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -z "$JAVA_HOME" ] ; then
|
||||
if [ -r /etc/gentoo-release ] ; then
|
||||
JAVA_HOME=`java-config --jre-home`
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$M2_HOME" ] ; then
|
||||
## resolve links - $0 may be a link to maven's home
|
||||
PRG="$0"
|
||||
|
||||
# need this for relative symlinks
|
||||
while [ -h "$PRG" ] ; do
|
||||
ls=`ls -ld "$PRG"`
|
||||
link=`expr "$ls" : '.*-> \(.*\)$'`
|
||||
if expr "$link" : '/.*' > /dev/null; then
|
||||
PRG="$link"
|
||||
else
|
||||
PRG="`dirname "$PRG"`/$link"
|
||||
fi
|
||||
done
|
||||
|
||||
saveddir=`pwd`
|
||||
|
||||
M2_HOME=`dirname "$PRG"`/..
|
||||
|
||||
# make it fully qualified
|
||||
M2_HOME=`cd "$M2_HOME" && pwd`
|
||||
|
||||
cd "$saveddir"
|
||||
# echo Using m2 at $M2_HOME
|
||||
fi
|
||||
|
||||
# For Cygwin, ensure paths are in UNIX format before anything is touched
|
||||
if $cygwin ; then
|
||||
[ -n "$M2_HOME" ] &&
|
||||
M2_HOME=`cygpath --unix "$M2_HOME"`
|
||||
[ -n "$JAVA_HOME" ] &&
|
||||
JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
|
||||
[ -n "$CLASSPATH" ] &&
|
||||
CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
|
||||
fi
|
||||
|
||||
# For Mingw, ensure paths are in UNIX format before anything is touched
|
||||
if $mingw ; then
|
||||
[ -n "$M2_HOME" ] &&
|
||||
M2_HOME="`(cd "$M2_HOME"; pwd)`"
|
||||
[ -n "$JAVA_HOME" ] &&
|
||||
JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
|
||||
fi
|
||||
|
||||
if [ -z "$JAVA_HOME" ]; then
|
||||
javaExecutable="`which javac`"
|
||||
if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
|
||||
# readlink(1) is not available as standard on Solaris 10.
|
||||
readLink=`which readlink`
|
||||
if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
|
||||
if $darwin ; then
|
||||
javaHome="`dirname \"$javaExecutable\"`"
|
||||
javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
|
||||
else
|
||||
javaExecutable="`readlink -f \"$javaExecutable\"`"
|
||||
fi
|
||||
javaHome="`dirname \"$javaExecutable\"`"
|
||||
javaHome=`expr "$javaHome" : '\(.*\)/bin'`
|
||||
JAVA_HOME="$javaHome"
|
||||
export JAVA_HOME
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$JAVACMD" ] ; then
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD="$JAVA_HOME/jre/sh/java"
|
||||
else
|
||||
JAVACMD="$JAVA_HOME/bin/java"
|
||||
fi
|
||||
else
|
||||
JAVACMD="`which java`"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
echo "Error: JAVA_HOME is not defined correctly." >&2
|
||||
echo " We cannot execute $JAVACMD" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$JAVA_HOME" ] ; then
|
||||
echo "Warning: JAVA_HOME environment variable is not set."
|
||||
fi
|
||||
|
||||
CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
|
||||
|
||||
# traverses directory structure from process work directory to filesystem root
|
||||
# first directory with .mvn subdirectory is considered project base directory
|
||||
find_maven_basedir() {
|
||||
|
||||
if [ -z "$1" ]
|
||||
then
|
||||
echo "Path not specified to find_maven_basedir"
|
||||
return 1
|
||||
fi
|
||||
|
||||
basedir="$1"
|
||||
wdir="$1"
|
||||
while [ "$wdir" != '/' ] ; do
|
||||
if [ -d "$wdir"/.mvn ] ; then
|
||||
basedir=$wdir
|
||||
break
|
||||
fi
|
||||
# workaround for JBEAP-8937 (on Solaris 10/Sparc)
|
||||
if [ -d "${wdir}" ]; then
|
||||
wdir=`cd "$wdir/.."; pwd`
|
||||
fi
|
||||
# end of workaround
|
||||
done
|
||||
echo "${basedir}"
|
||||
}
|
||||
|
||||
# concatenates all lines of a file
|
||||
concat_lines() {
|
||||
if [ -f "$1" ]; then
|
||||
echo "$(tr -s '\n' ' ' < "$1")"
|
||||
fi
|
||||
}
|
||||
|
||||
BASE_DIR=`find_maven_basedir "$(pwd)"`
|
||||
if [ -z "$BASE_DIR" ]; then
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
##########################################################################################
|
||||
# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
|
||||
# This allows using the maven wrapper in projects that prohibit checking in binary data.
|
||||
##########################################################################################
|
||||
if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
|
||||
if [ "$MVNW_VERBOSE" = true ]; then
|
||||
echo "Found .mvn/wrapper/maven-wrapper.jar"
|
||||
fi
|
||||
else
|
||||
if [ "$MVNW_VERBOSE" = true ]; then
|
||||
echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
|
||||
fi
|
||||
if [ -n "$MVNW_REPOURL" ]; then
|
||||
jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
|
||||
else
|
||||
jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
|
||||
fi
|
||||
while IFS="=" read key value; do
|
||||
case "$key" in (wrapperUrl) jarUrl="$value"; break ;;
|
||||
esac
|
||||
done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
|
||||
if [ "$MVNW_VERBOSE" = true ]; then
|
||||
echo "Downloading from: $jarUrl"
|
||||
fi
|
||||
wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
|
||||
if $cygwin; then
|
||||
wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"`
|
||||
fi
|
||||
|
||||
if command -v wget > /dev/null; then
|
||||
if [ "$MVNW_VERBOSE" = true ]; then
|
||||
echo "Found wget ... using wget"
|
||||
fi
|
||||
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
|
||||
wget "$jarUrl" -O "$wrapperJarPath"
|
||||
else
|
||||
wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath"
|
||||
fi
|
||||
elif command -v curl > /dev/null; then
|
||||
if [ "$MVNW_VERBOSE" = true ]; then
|
||||
echo "Found curl ... using curl"
|
||||
fi
|
||||
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
|
||||
curl -o "$wrapperJarPath" "$jarUrl" -f
|
||||
else
|
||||
curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f
|
||||
fi
|
||||
|
||||
else
|
||||
if [ "$MVNW_VERBOSE" = true ]; then
|
||||
echo "Falling back to using Java to download"
|
||||
fi
|
||||
javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
|
||||
# For Cygwin, switch paths to Windows format before running javac
|
||||
if $cygwin; then
|
||||
javaClass=`cygpath --path --windows "$javaClass"`
|
||||
fi
|
||||
if [ -e "$javaClass" ]; then
|
||||
if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
|
||||
if [ "$MVNW_VERBOSE" = true ]; then
|
||||
echo " - Compiling MavenWrapperDownloader.java ..."
|
||||
fi
|
||||
# Compiling the Java class
|
||||
("$JAVA_HOME/bin/javac" "$javaClass")
|
||||
fi
|
||||
if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
|
||||
# Running the downloader
|
||||
if [ "$MVNW_VERBOSE" = true ]; then
|
||||
echo " - Running MavenWrapperDownloader.java ..."
|
||||
fi
|
||||
("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
##########################################################################################
|
||||
# End of extension
|
||||
##########################################################################################
|
||||
|
||||
export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
|
||||
if [ "$MVNW_VERBOSE" = true ]; then
|
||||
echo $MAVEN_PROJECTBASEDIR
|
||||
fi
|
||||
MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
|
||||
|
||||
# For Cygwin, switch paths to Windows format before running java
|
||||
if $cygwin; then
|
||||
[ -n "$M2_HOME" ] &&
|
||||
M2_HOME=`cygpath --path --windows "$M2_HOME"`
|
||||
[ -n "$JAVA_HOME" ] &&
|
||||
JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
|
||||
[ -n "$CLASSPATH" ] &&
|
||||
CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
|
||||
[ -n "$MAVEN_PROJECTBASEDIR" ] &&
|
||||
MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
|
||||
fi
|
||||
|
||||
# Provide a "standardized" way to retrieve the CLI args that will
|
||||
# work with both Windows and non-Windows executions.
|
||||
MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@"
|
||||
export MAVEN_CMD_LINE_ARGS
|
||||
|
||||
WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
|
||||
|
||||
exec "$JAVACMD" \
|
||||
$MAVEN_OPTS \
|
||||
-classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
|
||||
"-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
|
||||
${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"
|
||||
182
binders/kafka-binder/mvnw.cmd
vendored
182
binders/kafka-binder/mvnw.cmd
vendored
@@ -1,182 +0,0 @@
|
||||
@REM ----------------------------------------------------------------------------
|
||||
@REM Licensed to the Apache Software Foundation (ASF) under one
|
||||
@REM or more contributor license agreements. See the NOTICE file
|
||||
@REM distributed with this work for additional information
|
||||
@REM regarding copyright ownership. The ASF licenses this file
|
||||
@REM to you under the Apache License, Version 2.0 (the
|
||||
@REM "License"); you may not use this file except in compliance
|
||||
@REM with the License. You may obtain a copy of the License at
|
||||
@REM
|
||||
@REM http://www.apache.org/licenses/LICENSE-2.0
|
||||
@REM
|
||||
@REM Unless required by applicable law or agreed to in writing,
|
||||
@REM software distributed under the License is distributed on an
|
||||
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
@REM KIND, either express or implied. See the License for the
|
||||
@REM specific language governing permissions and limitations
|
||||
@REM under the License.
|
||||
@REM ----------------------------------------------------------------------------
|
||||
|
||||
@REM ----------------------------------------------------------------------------
|
||||
@REM Maven Start Up Batch script
|
||||
@REM
|
||||
@REM Required ENV vars:
|
||||
@REM JAVA_HOME - location of a JDK home dir
|
||||
@REM
|
||||
@REM Optional ENV vars
|
||||
@REM M2_HOME - location of maven2's installed home dir
|
||||
@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
|
||||
@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
|
||||
@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
|
||||
@REM e.g. to debug Maven itself, use
|
||||
@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
|
||||
@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
|
||||
@REM ----------------------------------------------------------------------------
|
||||
|
||||
@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
|
||||
@echo off
|
||||
@REM set title of command window
|
||||
title %0
|
||||
@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
|
||||
@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
|
||||
|
||||
@REM set %HOME% to equivalent of $HOME
|
||||
if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
|
||||
|
||||
@REM Execute a user defined script before this one
|
||||
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
|
||||
@REM check for pre script, once with legacy .bat ending and once with .cmd ending
|
||||
if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
|
||||
if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
|
||||
:skipRcPre
|
||||
|
||||
@setlocal
|
||||
|
||||
set ERROR_CODE=0
|
||||
|
||||
@REM To isolate internal variables from possible post scripts, we use another setlocal
|
||||
@setlocal
|
||||
|
||||
@REM ==== START VALIDATION ====
|
||||
if not "%JAVA_HOME%" == "" goto OkJHome
|
||||
|
||||
echo.
|
||||
echo Error: JAVA_HOME not found in your environment. >&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the >&2
|
||||
echo location of your Java installation. >&2
|
||||
echo.
|
||||
goto error
|
||||
|
||||
:OkJHome
|
||||
if exist "%JAVA_HOME%\bin\java.exe" goto init
|
||||
|
||||
echo.
|
||||
echo Error: JAVA_HOME is set to an invalid directory. >&2
|
||||
echo JAVA_HOME = "%JAVA_HOME%" >&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the >&2
|
||||
echo location of your Java installation. >&2
|
||||
echo.
|
||||
goto error
|
||||
|
||||
@REM ==== END VALIDATION ====
|
||||
|
||||
:init
|
||||
|
||||
@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
|
||||
@REM Fallback to current working directory if not found.
|
||||
|
||||
set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
|
||||
IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
|
||||
|
||||
set EXEC_DIR=%CD%
|
||||
set WDIR=%EXEC_DIR%
|
||||
:findBaseDir
|
||||
IF EXIST "%WDIR%"\.mvn goto baseDirFound
|
||||
cd ..
|
||||
IF "%WDIR%"=="%CD%" goto baseDirNotFound
|
||||
set WDIR=%CD%
|
||||
goto findBaseDir
|
||||
|
||||
:baseDirFound
|
||||
set MAVEN_PROJECTBASEDIR=%WDIR%
|
||||
cd "%EXEC_DIR%"
|
||||
goto endDetectBaseDir
|
||||
|
||||
:baseDirNotFound
|
||||
set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
|
||||
cd "%EXEC_DIR%"
|
||||
|
||||
:endDetectBaseDir
|
||||
|
||||
IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
|
||||
|
||||
@setlocal EnableExtensions EnableDelayedExpansion
|
||||
for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
|
||||
@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
|
||||
|
||||
:endReadAdditionalConfig
|
||||
|
||||
SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
|
||||
set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
|
||||
set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
|
||||
|
||||
set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
|
||||
|
||||
FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
|
||||
IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B
|
||||
)
|
||||
|
||||
@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
|
||||
@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
|
||||
if exist %WRAPPER_JAR% (
|
||||
if "%MVNW_VERBOSE%" == "true" (
|
||||
echo Found %WRAPPER_JAR%
|
||||
)
|
||||
) else (
|
||||
if not "%MVNW_REPOURL%" == "" (
|
||||
SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
|
||||
)
|
||||
if "%MVNW_VERBOSE%" == "true" (
|
||||
echo Couldn't find %WRAPPER_JAR%, downloading it ...
|
||||
echo Downloading from: %DOWNLOAD_URL%
|
||||
)
|
||||
|
||||
powershell -Command "&{"^
|
||||
"$webclient = new-object System.Net.WebClient;"^
|
||||
"if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
|
||||
"$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
|
||||
"}"^
|
||||
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^
|
||||
"}"
|
||||
if "%MVNW_VERBOSE%" == "true" (
|
||||
echo Finished downloading %WRAPPER_JAR%
|
||||
)
|
||||
)
|
||||
@REM End of extension
|
||||
|
||||
@REM Provide a "standardized" way to retrieve the CLI args that will
|
||||
@REM work with both Windows and non-Windows executions.
|
||||
set MAVEN_CMD_LINE_ARGS=%*
|
||||
|
||||
%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
|
||||
if ERRORLEVEL 1 goto error
|
||||
goto end
|
||||
|
||||
:error
|
||||
set ERROR_CODE=1
|
||||
|
||||
:end
|
||||
@endlocal & set ERROR_CODE=%ERROR_CODE%
|
||||
|
||||
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
|
||||
@REM check for post script, once with legacy .bat ending and once with .cmd ending
|
||||
if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
|
||||
if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
|
||||
:skipRcPost
|
||||
|
||||
@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
|
||||
if "%MAVEN_BATCH_PAUSE%" == "on" pause
|
||||
|
||||
if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
|
||||
|
||||
exit /B %ERROR_CODE%
|
||||
@@ -1,182 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>spring-cloud-stream-binder-kafka-parent</artifactId>
|
||||
<version>4.1.0-SNAPSHOT</version>
|
||||
<packaging>pom</packaging>
|
||||
<parent>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-binders</artifactId>
|
||||
<version>4.1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<scm>
|
||||
<url>https://github.com/spring-cloud/spring-cloud-stream-binder-kafka</url>
|
||||
<connection>scm:git:git://github.com/spring-cloud/spring-cloud-stream-binder-kafka.git
|
||||
</connection>
|
||||
<developerConnection>
|
||||
scm:git:ssh://git@github.com/spring-cloud/spring-cloud-stream-binder-kafka.git
|
||||
</developerConnection>
|
||||
<tag>HEAD</tag>
|
||||
</scm>
|
||||
<properties>
|
||||
<java.version>17</java.version>
|
||||
<maven-checkstyle-plugin.failsOnError>true</maven-checkstyle-plugin.failsOnError>
|
||||
<maven-checkstyle-plugin.failsOnViolation>true</maven-checkstyle-plugin.failsOnViolation>
|
||||
<maven-checkstyle-plugin.includeTestSourceDirectory>true</maven-checkstyle-plugin.includeTestSourceDirectory>
|
||||
</properties>
|
||||
<modules>
|
||||
<module>spring-cloud-stream-binder-kafka</module>
|
||||
<module>spring-cloud-starter-stream-kafka</module>
|
||||
<module>spring-cloud-stream-binder-kafka-core</module>
|
||||
<module>spring-cloud-stream-binder-kafka-streams</module>
|
||||
<module>spring-cloud-stream-binder-kafka-reactive</module>
|
||||
</modules>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.junit.vintage</groupId>
|
||||
<artifactId>junit-vintage-engine</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<!-- <version>1.7</version>-->
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<configuration>
|
||||
<quiet>true</quiet>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>3.0.0-M7</version><!-- @releaser:version-check-off -->
|
||||
<configuration>
|
||||
<redirectTestOutputToFile>true</redirectTestOutputToFile>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>${maven-compiler-plugin.version}</version>
|
||||
<configuration>
|
||||
<source>${java.version}</source>
|
||||
<target>${java.version}</target>
|
||||
<compilerArgument>-parameters</compilerArgument>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>spring</id>
|
||||
|
||||
|
||||
</profile>
|
||||
<profile>
|
||||
<id>coverage</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>env.TRAVIS</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.7.9</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>spring-snapshots</id>
|
||||
<name>Spring Snapshots</name>
|
||||
<url>https://repo.spring.io/libs-snapshot-local</url>
|
||||
</repository>
|
||||
<repository>
|
||||
<id>spring-milestones</id>
|
||||
<name>Spring milestones</name>
|
||||
<url>https://repo.spring.io/libs-milestone-local</url>
|
||||
</repository>
|
||||
<repository>
|
||||
<id>rsocket-snapshots</id>
|
||||
<name>RSocket Snapshots</name>
|
||||
<url>https://oss.jfrog.org/oss-snapshot-local</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
<repository>
|
||||
<id>spring-releases</id>
|
||||
<name>Spring Releases</name>
|
||||
<url>https://repo.spring.io/release</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
<pluginRepositories>
|
||||
<pluginRepository>
|
||||
<id>spring-snapshots</id>
|
||||
<name>Spring Snapshots</name>
|
||||
<url>https://repo.spring.io/snapshot</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
</pluginRepository>
|
||||
<pluginRepository>
|
||||
<id>spring-milestones</id>
|
||||
<name>Spring Milestones</name>
|
||||
<url>https://repo.spring.io/milestone</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</pluginRepository>
|
||||
<pluginRepository>
|
||||
<id>spring-releases</id>
|
||||
<name>Spring Releases</name>
|
||||
<url>https://repo.spring.io/release</url>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
<reporting>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</reporting>
|
||||
</project>
|
||||
@@ -1,26 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-binder-kafka-parent</artifactId>
|
||||
<version>4.1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>spring-cloud-starter-stream-kafka</artifactId>
|
||||
<description>Spring Cloud Starter Stream Kafka</description>
|
||||
<url>https://projects.spring.io/spring-cloud</url>
|
||||
<organization>
|
||||
<name>Pivotal Software, Inc.</name>
|
||||
<url>https://www.spring.io</url>
|
||||
</organization>
|
||||
<properties>
|
||||
<main.basedir>${basedir}/../..</main.basedir>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-binder-kafka</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -1,49 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-binder-kafka-parent</artifactId>
|
||||
<version>4.1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>spring-cloud-stream-binder-kafka-core</artifactId>
|
||||
<description>Spring Cloud Stream Kafka Binder Core</description>
|
||||
<url>https://projects.spring.io/spring-cloud</url>
|
||||
<organization>
|
||||
<name>Pivotal Software, Inc.</name>
|
||||
<url>https://www.spring.io</url>
|
||||
</organization>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.integration</groupId>
|
||||
<artifactId>spring-integration-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-actuator</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-configuration-processor</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.kafka</groupId>
|
||||
<artifactId>spring-kafka-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -1,205 +0,0 @@
|
||||
/*
|
||||
* Copyright 2023-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.common;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import org.apache.kafka.clients.consumer.Consumer;
|
||||
import org.apache.kafka.common.PartitionInfo;
|
||||
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.boot.actuate.health.AbstractHealthIndicator;
|
||||
import org.springframework.boot.actuate.health.Health;
|
||||
import org.springframework.boot.actuate.health.Status;
|
||||
import org.springframework.boot.actuate.health.StatusAggregator;
|
||||
import org.springframework.kafka.core.ConsumerFactory;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
/**
|
||||
* Base class that abstracts the common health indicator details for the various Kafka binder flavors.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @since 4.1.0
|
||||
*/
|
||||
public abstract class AbstractKafkaBinderHealthIndicator extends AbstractHealthIndicator implements DisposableBean {
|
||||
|
||||
private static final int DEFAULT_TIMEOUT = 60;
|
||||
|
||||
protected int timeout = DEFAULT_TIMEOUT;
|
||||
|
||||
private final ExecutorService executor;
|
||||
|
||||
protected Consumer<?, ?> metadataConsumer;
|
||||
|
||||
protected boolean considerDownWhenAnyPartitionHasNoLeader;
|
||||
|
||||
private final ConsumerFactory<?, ?> consumerFactory;
|
||||
|
||||
public AbstractKafkaBinderHealthIndicator(ConsumerFactory<?, ?> consumerFactory) {
|
||||
this.consumerFactory = consumerFactory;
|
||||
this.executor = createHealthBinderExecutorService();
|
||||
Assert.notNull(this.executor, "The health indicator executor service must not be null");
|
||||
}
|
||||
|
||||
protected abstract Map<String, TopicInformation> getTopicsInUse();
|
||||
|
||||
protected abstract Health buildBinderSpecificHealthDetails();
|
||||
|
||||
protected abstract ExecutorService createHealthBinderExecutorService();
|
||||
|
||||
private void initMetadataConsumer() {
|
||||
if (this.metadataConsumer == null) {
|
||||
this.metadataConsumer = this.consumerFactory.createConsumer();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
executor.shutdown();
|
||||
if (this.metadataConsumer != null) {
|
||||
this.metadataConsumer.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doHealthCheck(Health.Builder builder) throws Exception {
|
||||
Health topicsHealth = safelyBuildTopicsHealth();
|
||||
Health listenerContainersHealth = buildBinderSpecificHealthDetails();
|
||||
merge(topicsHealth, listenerContainersHealth, builder);
|
||||
}
|
||||
|
||||
protected Health safelyBuildTopicsHealth() {
|
||||
Future<Health> future = executor.submit(this::buildTopicsHealth);
|
||||
try {
|
||||
return future.get(this.timeout, TimeUnit.SECONDS);
|
||||
}
|
||||
catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
return Health.down()
|
||||
.withDetail("Interrupted while waiting for partition information in",
|
||||
this.timeout + " seconds")
|
||||
.build();
|
||||
}
|
||||
catch (ExecutionException ex) {
|
||||
return Health.down(ex).build();
|
||||
}
|
||||
catch (TimeoutException ex) {
|
||||
return Health.down().withDetail("Failed to retrieve partition information in",
|
||||
this.timeout + " seconds").build();
|
||||
}
|
||||
}
|
||||
|
||||
private Health buildTopicsHealth() {
|
||||
try {
|
||||
initMetadataConsumer();
|
||||
Set<String> downMessages = new HashSet<>();
|
||||
Set<String> checkedTopics = new HashSet<>();
|
||||
final Map<String, TopicInformation> topicsInUse = getTopicsInUse();
|
||||
if (topicsInUse.isEmpty()) {
|
||||
try {
|
||||
this.metadataConsumer.listTopics(Duration.ofSeconds(this.timeout));
|
||||
}
|
||||
catch (Exception e) {
|
||||
return Health.down().withDetail("No topic information available",
|
||||
"Kafka broker is not reachable").build();
|
||||
}
|
||||
return Health.unknown().withDetail("No bindings found",
|
||||
"Kafka binder may not be bound to destinations on the broker").build();
|
||||
}
|
||||
else {
|
||||
for (String topic : topicsInUse.keySet()) {
|
||||
TopicInformation topicInformation = topicsInUse
|
||||
.get(topic);
|
||||
if (!topicInformation.isTopicPattern()) {
|
||||
List<PartitionInfo> partitionInfos = this.metadataConsumer
|
||||
.partitionsFor(topic);
|
||||
for (PartitionInfo partitionInfo : partitionInfos) {
|
||||
if (topicInformation.partitionInfos()
|
||||
.contains(partitionInfo)
|
||||
&& partitionInfo.leader() == null ||
|
||||
(partitionInfo.leader() != null && partitionInfo.leader().id() == -1)) {
|
||||
downMessages.add(partitionInfo.toString());
|
||||
}
|
||||
else if (this.considerDownWhenAnyPartitionHasNoLeader &&
|
||||
partitionInfo.leader() == null || (partitionInfo.leader() != null && partitionInfo.leader().id() == -1)) {
|
||||
downMessages.add(partitionInfo.toString());
|
||||
}
|
||||
}
|
||||
checkedTopics.add(topic);
|
||||
}
|
||||
else {
|
||||
try {
|
||||
// Since destination is a pattern, all we are doing is just to make sure that
|
||||
// we can connect to the cluster and query the topics.
|
||||
this.metadataConsumer.listTopics(Duration.ofSeconds(this.timeout));
|
||||
}
|
||||
catch (Exception ex) {
|
||||
return Health.down()
|
||||
.withDetail("Cluster not connected",
|
||||
"Destination provided is a pattern, but cannot connect to the cluster for any verification")
|
||||
.build();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (downMessages.isEmpty()) {
|
||||
return Health.up().withDetail("topicsInUse", checkedTopics).build();
|
||||
}
|
||||
else {
|
||||
return Health.down()
|
||||
.withDetail("Following partitions in use have no leaders: ",
|
||||
downMessages.toString())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
catch (Exception ex) {
|
||||
return Health.down(ex).build();
|
||||
}
|
||||
}
|
||||
|
||||
private void merge(Health topicsHealth, Health listenerContainersHealth, Health.Builder builder) {
|
||||
Status aggregatedStatus = StatusAggregator.getDefault()
|
||||
.getAggregateStatus(topicsHealth.getStatus(), listenerContainersHealth.getStatus());
|
||||
Map<String, Object> aggregatedDetails = new HashMap<>();
|
||||
aggregatedDetails.putAll(topicsHealth.getDetails());
|
||||
aggregatedDetails.putAll(listenerContainersHealth.getDetails());
|
||||
builder.status(aggregatedStatus).withDetails(aggregatedDetails);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the timeout in seconds to retrieve health information.
|
||||
*
|
||||
* @param timeout the timeout - default 60.
|
||||
*/
|
||||
public void setTimeout(int timeout) {
|
||||
this.timeout = timeout;
|
||||
}
|
||||
|
||||
public void setConsiderDownWhenAnyPartitionHasNoLeader(boolean considerDownWhenAnyPartitionHasNoLeader) {
|
||||
this.considerDownWhenAnyPartitionHasNoLeader = considerDownWhenAnyPartitionHasNoLeader;
|
||||
}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
/*
|
||||
* Copyright 2017-2019 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.common;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.kafka.common.serialization.ByteArrayDeserializer;
|
||||
import org.apache.kafka.common.serialization.ByteArraySerializer;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.env.EnvironmentPostProcessor;
|
||||
import org.springframework.core.env.ConfigurableEnvironment;
|
||||
import org.springframework.core.env.MapPropertySource;
|
||||
|
||||
/**
|
||||
* An {@link EnvironmentPostProcessor} that sets some common configuration properties (log
|
||||
* config etc.,) for Kafka binder.
|
||||
*
|
||||
* @author Ilayaperumal Gopinathan
|
||||
*/
|
||||
public class KafkaBinderEnvironmentPostProcessor implements EnvironmentPostProcessor {
|
||||
|
||||
private static final String SPRING_KAFKA = "spring.kafka";
|
||||
|
||||
private static final String SPRING_KAFKA_PRODUCER = SPRING_KAFKA + ".producer";
|
||||
|
||||
private static final String SPRING_KAFKA_CONSUMER = SPRING_KAFKA + ".consumer";
|
||||
|
||||
private static final String SPRING_KAFKA_PRODUCER_KEY_SERIALIZER = SPRING_KAFKA_PRODUCER
|
||||
+ "." + "keySerializer";
|
||||
|
||||
private static final String SPRING_KAFKA_PRODUCER_VALUE_SERIALIZER = SPRING_KAFKA_PRODUCER
|
||||
+ "." + "valueSerializer";
|
||||
|
||||
private static final String SPRING_KAFKA_CONSUMER_KEY_DESERIALIZER = SPRING_KAFKA_CONSUMER
|
||||
+ "." + "keyDeserializer";
|
||||
|
||||
private static final String SPRING_KAFKA_CONSUMER_VALUE_DESERIALIZER = SPRING_KAFKA_CONSUMER
|
||||
+ "." + "valueDeserializer";
|
||||
|
||||
private static final String KAFKA_BINDER_DEFAULT_PROPERTIES = "kafkaBinderDefaultProperties";
|
||||
|
||||
@Override
|
||||
public void postProcessEnvironment(ConfigurableEnvironment environment,
|
||||
SpringApplication application) {
|
||||
if (!environment.getPropertySources().contains(KAFKA_BINDER_DEFAULT_PROPERTIES)) {
|
||||
Map<String, Object> kafkaBinderDefaultProperties = new HashMap<>();
|
||||
kafkaBinderDefaultProperties.put("logging.level.org.I0Itec.zkclient",
|
||||
"ERROR");
|
||||
kafkaBinderDefaultProperties.put("logging.level.kafka.server.KafkaConfig",
|
||||
"ERROR");
|
||||
kafkaBinderDefaultProperties
|
||||
.put("logging.level.kafka.admin.AdminClient.AdminConfig", "ERROR");
|
||||
kafkaBinderDefaultProperties.put(SPRING_KAFKA_PRODUCER_KEY_SERIALIZER,
|
||||
ByteArraySerializer.class.getName());
|
||||
kafkaBinderDefaultProperties.put(SPRING_KAFKA_PRODUCER_VALUE_SERIALIZER,
|
||||
ByteArraySerializer.class.getName());
|
||||
kafkaBinderDefaultProperties.put(SPRING_KAFKA_CONSUMER_KEY_DESERIALIZER,
|
||||
ByteArrayDeserializer.class.getName());
|
||||
kafkaBinderDefaultProperties.put(SPRING_KAFKA_CONSUMER_VALUE_DESERIALIZER,
|
||||
ByteArrayDeserializer.class.getName());
|
||||
environment.getPropertySources().addLast(new MapPropertySource(
|
||||
KAFKA_BINDER_DEFAULT_PROPERTIES, kafkaBinderDefaultProperties));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
/*
|
||||
* Copyright 2023-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.common;
|
||||
|
||||
import java.util.Collection;
|
||||
|
||||
import org.apache.kafka.common.PartitionInfo;
|
||||
|
||||
/**
|
||||
* Record to capture topic information for various binder related tasks.
|
||||
*
|
||||
* @param consumerGroup consumer group for the consumer
|
||||
* @param partitionInfos collection of {@link PartitionInfo}
|
||||
* @param isTopicPattern if the topic is specified as a pattern
|
||||
*
|
||||
* @author Soby Chacko (and previous authors before refactoring).
|
||||
*/
|
||||
public record TopicInformation(String consumerGroup, Collection<PartitionInfo> partitionInfos, boolean isTopicPattern) {
|
||||
|
||||
public boolean isConsumerTopic() {
|
||||
return this.consumerGroup != null;
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
/*
|
||||
* Copyright 2016-2018 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.properties;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.security.auth.login.AppConfigurationEntry;
|
||||
|
||||
import org.springframework.kafka.security.jaas.KafkaJaasLoginModuleInitializer;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
/**
|
||||
* Contains properties for setting up an {@link AppConfigurationEntry} that can be used
|
||||
* for the Kafka or Zookeeper client.
|
||||
*
|
||||
* @author Marius Bogoevici
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
public class JaasLoginModuleConfiguration {
|
||||
|
||||
private String loginModule = "com.sun.security.auth.module.Krb5LoginModule";
|
||||
|
||||
private KafkaJaasLoginModuleInitializer.ControlFlag controlFlag = KafkaJaasLoginModuleInitializer.ControlFlag.REQUIRED;
|
||||
|
||||
private Map<String, String> options = new HashMap<>();
|
||||
|
||||
public String getLoginModule() {
|
||||
return this.loginModule;
|
||||
}
|
||||
|
||||
public void setLoginModule(String loginModule) {
|
||||
Assert.notNull(loginModule, "cannot be null");
|
||||
this.loginModule = loginModule;
|
||||
}
|
||||
|
||||
public KafkaJaasLoginModuleInitializer.ControlFlag getControlFlag() {
|
||||
return this.controlFlag;
|
||||
}
|
||||
|
||||
public void setControlFlag(String controlFlag) {
|
||||
Assert.notNull(controlFlag, "cannot be null");
|
||||
this.controlFlag = KafkaJaasLoginModuleInitializer.ControlFlag
|
||||
.valueOf(controlFlag.toUpperCase());
|
||||
}
|
||||
|
||||
public Map<String, String> getOptions() {
|
||||
return this.options;
|
||||
}
|
||||
|
||||
public void setOptions(Map<String, String> options) {
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,723 +0,0 @@
|
||||
/*
|
||||
* Copyright 2015-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.properties;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.time.Duration;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import jakarta.validation.constraints.AssertTrue;
|
||||
import jakarta.validation.constraints.Min;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.kafka.clients.consumer.ConsumerConfig;
|
||||
import org.apache.kafka.clients.producer.ProducerConfig;
|
||||
|
||||
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
|
||||
import org.springframework.cloud.stream.binder.HeaderMode;
|
||||
import org.springframework.cloud.stream.binder.ProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties.CompressionType;
|
||||
import org.springframework.core.io.DefaultResourceLoader;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.expression.Expression;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.ObjectUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* Configuration properties for the Kafka binder. The properties in this class are
|
||||
* prefixed with <b>spring.cloud.stream.kafka.binder</b>.
|
||||
*
|
||||
* @author David Turanski
|
||||
* @author Ilayaperumal Gopinathan
|
||||
* @author Marius Bogoevici
|
||||
* @author Soby Chacko
|
||||
* @author Gary Russell
|
||||
* @author Rafal Zukowski
|
||||
* @author Aldo Sinanaj
|
||||
* @author Lukasz Kaminski
|
||||
* @author Chukwubuikem Ume-Ugwa
|
||||
* @author Nico Heller
|
||||
* @author Norbert Gyurian
|
||||
*/
|
||||
public class KafkaBinderConfigurationProperties {
|
||||
|
||||
private static final String DEFAULT_KAFKA_CONNECTION_STRING = "localhost:9092";
|
||||
|
||||
private final Log logger = LogFactory.getLog(getClass());
|
||||
|
||||
private final Transaction transaction = new Transaction();
|
||||
|
||||
private final Metrics metrics = new Metrics();
|
||||
|
||||
private final KafkaProperties kafkaProperties;
|
||||
|
||||
/**
|
||||
* Arbitrary kafka properties that apply to both producers and consumers.
|
||||
*/
|
||||
private Map<String, String> configuration = new HashMap<>();
|
||||
|
||||
/**
|
||||
* Arbitrary kafka consumer properties.
|
||||
*/
|
||||
private Map<String, String> consumerProperties = new HashMap<>();
|
||||
|
||||
/**
|
||||
* Arbitrary kafka producer properties.
|
||||
*/
|
||||
private Map<String, String> producerProperties = new HashMap<>();
|
||||
|
||||
private String[] brokers = new String[] { "localhost" };
|
||||
|
||||
private String defaultBrokerPort = "9092";
|
||||
|
||||
private String[] headers = new String[] {};
|
||||
|
||||
private boolean autoCreateTopics = true;
|
||||
|
||||
private boolean autoAlterTopics;
|
||||
|
||||
private boolean autoAddPartitions;
|
||||
|
||||
private boolean considerDownWhenAnyPartitionHasNoLeader;
|
||||
|
||||
private String requiredAcks = "1";
|
||||
|
||||
private short replicationFactor = -1;
|
||||
|
||||
private int minPartitionCount = 1;
|
||||
|
||||
/**
|
||||
* Time to wait to get partition information in seconds; default 60.
|
||||
*/
|
||||
private int healthTimeout = 60;
|
||||
|
||||
private JaasLoginModuleConfiguration jaas;
|
||||
|
||||
/**
|
||||
* The bean name of a custom header mapper to use instead of a
|
||||
* {@link org.springframework.kafka.support.DefaultKafkaHeaderMapper}.
|
||||
*/
|
||||
private String headerMapperBeanName;
|
||||
|
||||
/**
|
||||
* Time between retries after AuthorizationException is caught in
|
||||
* the ListenerContainer; defalt is null which disables retries.
|
||||
* For more info see: {@link org.springframework.kafka.listener.ConsumerProperties#setAuthorizationExceptionRetryInterval(java.time.Duration)}
|
||||
*/
|
||||
private Duration authorizationExceptionRetryInterval;
|
||||
|
||||
/**
|
||||
* When a certificate store location is not given as a local file system path, then the binder
|
||||
* converts the path to {@link org.springframework.core.io.Resource} resource, then copies
|
||||
* the resource from the original location to a location on
|
||||
* the filesystem. If this value is set, then this location is used, otherwise, the
|
||||
* certificate file is copied to the directory returned by java.io.tmpdir.
|
||||
*/
|
||||
private String certificateStoreDirectory;
|
||||
|
||||
/**
|
||||
* Enable Micrometer observation registry across all the bindings in the binder.
|
||||
*/
|
||||
private boolean enableObservation;
|
||||
|
||||
/**
|
||||
* Earlier, @Autowired on this constructor was necessary for all the properties to be discovered
|
||||
* and bound when running as a native application. However, now that Spring Boot fixed the underlying
|
||||
* issue, we are removing the @Autowired from the constructor. See these Boot issues for more details.
|
||||
* https://github.com/spring-projects/spring-boot/issues/34507
|
||||
* https://github.com/spring-projects/spring-boot/issues/35564
|
||||
*
|
||||
* @param kafkaProperties Spring Kafka properties autoconfigured by Spring Boot
|
||||
*/
|
||||
public KafkaBinderConfigurationProperties(KafkaProperties kafkaProperties) {
|
||||
Assert.notNull(kafkaProperties, "'kafkaProperties' cannot be null");
|
||||
this.kafkaProperties = kafkaProperties;
|
||||
}
|
||||
|
||||
public KafkaProperties getKafkaProperties() {
|
||||
return this.kafkaProperties;
|
||||
}
|
||||
|
||||
public Transaction getTransaction() {
|
||||
return this.transaction;
|
||||
}
|
||||
|
||||
public Metrics getMetrics() {
|
||||
return metrics;
|
||||
}
|
||||
|
||||
public String getKafkaConnectionString() {
|
||||
// We need to do a check on certificate file locations to see if they are given as non-local file system resources.
|
||||
// If that is the case, then we will copy them to a file system location and use those as the certificate locations.
|
||||
// This is due to a limitation in Kafka itself in which it doesn't allow reading certificate resources from non-local
|
||||
// file system locations e.g. classpath, http.
|
||||
// See this: https://issues.apache.org/jira/browse/KAFKA-7685
|
||||
// and this: https://cwiki.apache.org/confluence/display/KAFKA/KIP-398%3A+Support+reading+trust+store+from+classpath
|
||||
moveCertsToFileSystemIfNecessary();
|
||||
|
||||
return toConnectionString(this.brokers, this.defaultBrokerPort);
|
||||
}
|
||||
|
||||
private void moveCertsToFileSystemIfNecessary() {
|
||||
try {
|
||||
// broker certificates
|
||||
moveCertsIfApplicable("ssl.truststore.location");
|
||||
moveCertsIfApplicable("ssl.keystore.location");
|
||||
|
||||
// schema registry certificates
|
||||
moveCertsIfApplicable("schema.registry.ssl.truststore.location");
|
||||
moveCertsIfApplicable("schema.registry.ssl.keystore.location");
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private void moveCertsIfApplicable(String storeProperty) throws IOException {
|
||||
final String storeLocation = this.configuration.get(storeProperty);
|
||||
|
||||
// If the path is not defined, or it is a local file path do not move the file
|
||||
if (storeLocation != null && !checkIfFileExists(storeLocation)) {
|
||||
final String fileSystemLocation = moveCertToFileSystem(storeLocation, this.certificateStoreDirectory);
|
||||
// Overriding the value with absolute filesystem path.
|
||||
this.configuration.put(storeProperty, fileSystemLocation);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean checkIfFileExists(String path) {
|
||||
try {
|
||||
return Files.isRegularFile(Paths.get(path));
|
||||
}
|
||||
catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private String moveCertToFileSystem(String resourceLocation, String fileSystemLocation) throws IOException {
|
||||
File targetFile;
|
||||
final String tempDir = System.getProperty("java.io.tmpdir");
|
||||
Resource resource = new DefaultResourceLoader().getResource(resourceLocation);
|
||||
if (StringUtils.hasText(fileSystemLocation)) {
|
||||
final Path path = Paths.get(fileSystemLocation);
|
||||
if (!Files.exists(path) || !Files.isDirectory(path) || !Files.isWritable(path)) {
|
||||
logger.warn("The filesystem location to move the cert files (" + fileSystemLocation + ") " +
|
||||
"is not found or a directory that is writable. The system temp folder (java.io.tmpdir) will be used instead.");
|
||||
targetFile = new File(Paths.get(tempDir, resource.getFilename()).toString());
|
||||
}
|
||||
else {
|
||||
// the given location is verified to be a writable directory.
|
||||
targetFile = new File(Paths.get(fileSystemLocation, resource.getFilename()).toString());
|
||||
}
|
||||
}
|
||||
else {
|
||||
targetFile = new File(Paths.get(tempDir, resource.getFilename()).toString());
|
||||
}
|
||||
|
||||
try (InputStream inputStream = resource.getInputStream()) {
|
||||
Files.copy(inputStream, targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
return targetFile.getAbsolutePath();
|
||||
}
|
||||
|
||||
public String getDefaultKafkaConnectionString() {
|
||||
return DEFAULT_KAFKA_CONNECTION_STRING;
|
||||
}
|
||||
|
||||
public String[] getHeaders() {
|
||||
return this.headers;
|
||||
}
|
||||
|
||||
public String[] getBrokers() {
|
||||
return this.brokers;
|
||||
}
|
||||
|
||||
public void setBrokers(String... brokers) {
|
||||
this.brokers = brokers;
|
||||
}
|
||||
|
||||
public void setDefaultBrokerPort(String defaultBrokerPort) {
|
||||
this.defaultBrokerPort = defaultBrokerPort;
|
||||
}
|
||||
|
||||
public void setHeaders(String... headers) {
|
||||
this.headers = headers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts an array of host values to a comma-separated String. It will append the
|
||||
* default port value, if not already specified.
|
||||
* @param hosts host string
|
||||
* @param defaultPort port
|
||||
* @return formatted connection string
|
||||
*/
|
||||
private String toConnectionString(String[] hosts, String defaultPort) {
|
||||
String[] fullyFormattedHosts = new String[hosts.length];
|
||||
for (int i = 0; i < hosts.length; i++) {
|
||||
if (hosts[i].contains(":") || !StringUtils.hasText(defaultPort)) {
|
||||
fullyFormattedHosts[i] = hosts[i];
|
||||
}
|
||||
else {
|
||||
fullyFormattedHosts[i] = hosts[i] + ":" + defaultPort;
|
||||
}
|
||||
}
|
||||
return StringUtils.arrayToCommaDelimitedString(fullyFormattedHosts);
|
||||
}
|
||||
|
||||
public String getRequiredAcks() {
|
||||
return this.requiredAcks;
|
||||
}
|
||||
|
||||
public void setRequiredAcks(String requiredAcks) {
|
||||
this.requiredAcks = requiredAcks;
|
||||
}
|
||||
|
||||
public short getReplicationFactor() {
|
||||
return this.replicationFactor;
|
||||
}
|
||||
|
||||
public void setReplicationFactor(short replicationFactor) {
|
||||
this.replicationFactor = replicationFactor;
|
||||
}
|
||||
|
||||
public int getMinPartitionCount() {
|
||||
return this.minPartitionCount;
|
||||
}
|
||||
|
||||
public void setMinPartitionCount(int minPartitionCount) {
|
||||
this.minPartitionCount = minPartitionCount;
|
||||
}
|
||||
|
||||
public int getHealthTimeout() {
|
||||
return this.healthTimeout;
|
||||
}
|
||||
|
||||
public void setHealthTimeout(int healthTimeout) {
|
||||
this.healthTimeout = healthTimeout;
|
||||
}
|
||||
|
||||
public boolean isAutoCreateTopics() {
|
||||
return this.autoCreateTopics;
|
||||
}
|
||||
|
||||
public void setAutoCreateTopics(boolean autoCreateTopics) {
|
||||
this.autoCreateTopics = autoCreateTopics;
|
||||
}
|
||||
|
||||
public boolean isAutoAlterTopics() {
|
||||
return autoAlterTopics;
|
||||
}
|
||||
|
||||
public void setAutoAlterTopics(boolean autoAlterTopics) {
|
||||
this.autoAlterTopics = autoAlterTopics;
|
||||
}
|
||||
|
||||
public boolean isAutoAddPartitions() {
|
||||
return this.autoAddPartitions;
|
||||
}
|
||||
|
||||
public void setAutoAddPartitions(boolean autoAddPartitions) {
|
||||
this.autoAddPartitions = autoAddPartitions;
|
||||
}
|
||||
|
||||
public Map<String, String> getConfiguration() {
|
||||
return this.configuration;
|
||||
}
|
||||
|
||||
public void setConfiguration(Map<String, String> configuration) {
|
||||
this.configuration = configuration;
|
||||
}
|
||||
|
||||
public Map<String, String> getConsumerProperties() {
|
||||
return this.consumerProperties;
|
||||
}
|
||||
|
||||
public void setConsumerProperties(Map<String, String> consumerProperties) {
|
||||
Assert.notNull(consumerProperties, "'consumerProperties' cannot be null");
|
||||
this.consumerProperties = consumerProperties;
|
||||
}
|
||||
|
||||
public Map<String, String> getProducerProperties() {
|
||||
return this.producerProperties;
|
||||
}
|
||||
|
||||
public void setProducerProperties(Map<String, String> producerProperties) {
|
||||
Assert.notNull(producerProperties, "'producerProperties' cannot be null");
|
||||
this.producerProperties = producerProperties;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge boot consumer properties, general properties from
|
||||
* {@link #setConfiguration(Map)} that apply to consumers, properties from
|
||||
* {@link #setConsumerProperties(Map)}, in that order.
|
||||
* @return the merged properties.
|
||||
*/
|
||||
public Map<String, Object> mergedConsumerConfiguration() {
|
||||
Map<String, Object> consumerConfiguration = new HashMap<>();
|
||||
consumerConfiguration.putAll(this.kafkaProperties.buildConsumerProperties());
|
||||
// Copy configured binder properties that apply to consumers
|
||||
for (Map.Entry<String, String> configurationEntry : this.configuration
|
||||
.entrySet()) {
|
||||
if (ConsumerConfig.configNames().contains(configurationEntry.getKey())) {
|
||||
consumerConfiguration.put(configurationEntry.getKey(),
|
||||
configurationEntry.getValue());
|
||||
}
|
||||
}
|
||||
consumerConfiguration.putAll(this.consumerProperties);
|
||||
filterStreamManagedConfiguration(consumerConfiguration);
|
||||
// Override Spring Boot bootstrap server setting if left to default with the value
|
||||
// configured in the binder
|
||||
return getConfigurationWithBootstrapServer(consumerConfiguration,
|
||||
ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG);
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge boot producer properties, general properties from
|
||||
* {@link #setConfiguration(Map)} that apply to producers, properties from
|
||||
* {@link #setProducerProperties(Map)}, in that order.
|
||||
* @return the merged properties.
|
||||
*/
|
||||
public Map<String, Object> mergedProducerConfiguration() {
|
||||
Map<String, Object> producerConfiguration = new HashMap<>();
|
||||
producerConfiguration.putAll(this.kafkaProperties.buildProducerProperties());
|
||||
// Copy configured binder properties that apply to producers
|
||||
for (Map.Entry<String, String> configurationEntry : this.configuration
|
||||
.entrySet()) {
|
||||
if (ProducerConfig.configNames().contains(configurationEntry.getKey())) {
|
||||
producerConfiguration.put(configurationEntry.getKey(),
|
||||
configurationEntry.getValue());
|
||||
}
|
||||
}
|
||||
producerConfiguration.putAll(this.producerProperties);
|
||||
// Override Spring Boot bootstrap server setting if left to default with the value
|
||||
// configured in the binder
|
||||
return getConfigurationWithBootstrapServer(producerConfiguration,
|
||||
ProducerConfig.BOOTSTRAP_SERVERS_CONFIG);
|
||||
}
|
||||
|
||||
private void filterStreamManagedConfiguration(Map<String, Object> configuration) {
|
||||
if (configuration.containsKey(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)
|
||||
&& configuration.get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG).toString().equalsIgnoreCase("true")) {
|
||||
logger.warn(constructIgnoredConfigMessage(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG) +
|
||||
ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG + "=true is not supported by the Kafka binder");
|
||||
configuration.remove(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG);
|
||||
}
|
||||
if (configuration.containsKey(ConsumerConfig.GROUP_ID_CONFIG)) {
|
||||
logger.warn(constructIgnoredConfigMessage(ConsumerConfig.GROUP_ID_CONFIG) +
|
||||
"Use spring.cloud.stream.default.group or spring.cloud.stream.binding.<name>.group to specify " +
|
||||
"the group instead of " + ConsumerConfig.GROUP_ID_CONFIG);
|
||||
configuration.remove(ConsumerConfig.GROUP_ID_CONFIG);
|
||||
}
|
||||
}
|
||||
|
||||
private String constructIgnoredConfigMessage(String config) {
|
||||
return String.format("Ignoring provided value(s) for '%s'. ", config);
|
||||
}
|
||||
|
||||
private Map<String, Object> getConfigurationWithBootstrapServer(
|
||||
Map<String, Object> configuration, String bootstrapServersConfig) {
|
||||
final String kafkaConnectionString = getKafkaConnectionString();
|
||||
if (ObjectUtils.isEmpty(configuration.get(bootstrapServersConfig)) ||
|
||||
!kafkaConnectionString.equals("localhost:9092")) {
|
||||
configuration.put(bootstrapServersConfig, kafkaConnectionString);
|
||||
}
|
||||
return Collections.unmodifiableMap(configuration);
|
||||
}
|
||||
|
||||
public JaasLoginModuleConfiguration getJaas() {
|
||||
return this.jaas;
|
||||
}
|
||||
|
||||
public void setJaas(JaasLoginModuleConfiguration jaas) {
|
||||
this.jaas = jaas;
|
||||
}
|
||||
|
||||
public String getHeaderMapperBeanName() {
|
||||
return this.headerMapperBeanName;
|
||||
}
|
||||
|
||||
public void setHeaderMapperBeanName(String headerMapperBeanName) {
|
||||
this.headerMapperBeanName = headerMapperBeanName;
|
||||
}
|
||||
|
||||
public Duration getAuthorizationExceptionRetryInterval() {
|
||||
return authorizationExceptionRetryInterval;
|
||||
}
|
||||
|
||||
public void setAuthorizationExceptionRetryInterval(Duration authorizationExceptionRetryInterval) {
|
||||
this.authorizationExceptionRetryInterval = authorizationExceptionRetryInterval;
|
||||
}
|
||||
|
||||
public boolean isConsiderDownWhenAnyPartitionHasNoLeader() {
|
||||
return this.considerDownWhenAnyPartitionHasNoLeader;
|
||||
}
|
||||
|
||||
public void setConsiderDownWhenAnyPartitionHasNoLeader(boolean considerDownWhenAnyPartitionHasNoLeader) {
|
||||
this.considerDownWhenAnyPartitionHasNoLeader = considerDownWhenAnyPartitionHasNoLeader;
|
||||
}
|
||||
|
||||
public String getCertificateStoreDirectory() {
|
||||
return this.certificateStoreDirectory;
|
||||
}
|
||||
|
||||
public void setCertificateStoreDirectory(String certificateStoreDirectory) {
|
||||
this.certificateStoreDirectory = certificateStoreDirectory;
|
||||
}
|
||||
|
||||
public boolean isEnableObservation() {
|
||||
return this.enableObservation;
|
||||
}
|
||||
|
||||
public void setEnableObservation(boolean enableObservation) {
|
||||
this.enableObservation = enableObservation;
|
||||
}
|
||||
|
||||
/**
|
||||
* Domain class that models transaction capabilities in Kafka.
|
||||
*/
|
||||
public static class Transaction {
|
||||
|
||||
private final CombinedProducerProperties producer = new CombinedProducerProperties();
|
||||
|
||||
private String transactionIdPrefix;
|
||||
|
||||
public String getTransactionIdPrefix() {
|
||||
return this.transactionIdPrefix;
|
||||
}
|
||||
|
||||
public void setTransactionIdPrefix(String transactionIdPrefix) {
|
||||
this.transactionIdPrefix = transactionIdPrefix;
|
||||
}
|
||||
|
||||
public CombinedProducerProperties getProducer() {
|
||||
return this.producer;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* An combination of {@link ProducerProperties} and {@link KafkaProducerProperties} so
|
||||
* that common and kafka-specific properties can be set for the transactional
|
||||
* producer.
|
||||
*
|
||||
* @since 2.1
|
||||
*/
|
||||
public static class CombinedProducerProperties {
|
||||
|
||||
private final ProducerProperties producerProperties = new ProducerProperties();
|
||||
|
||||
private final KafkaProducerProperties kafkaProducerProperties = new KafkaProducerProperties();
|
||||
|
||||
public Expression getPartitionKeyExpression() {
|
||||
return this.producerProperties.getPartitionKeyExpression();
|
||||
}
|
||||
|
||||
public void setPartitionKeyExpression(Expression partitionKeyExpression) {
|
||||
this.producerProperties.setPartitionKeyExpression(partitionKeyExpression);
|
||||
}
|
||||
|
||||
public boolean isPartitioned() {
|
||||
return this.producerProperties.isPartitioned();
|
||||
}
|
||||
|
||||
public Expression getPartitionSelectorExpression() {
|
||||
return this.producerProperties.getPartitionSelectorExpression();
|
||||
}
|
||||
|
||||
public void setPartitionSelectorExpression(
|
||||
Expression partitionSelectorExpression) {
|
||||
this.producerProperties
|
||||
.setPartitionSelectorExpression(partitionSelectorExpression);
|
||||
}
|
||||
|
||||
public @Min(value = 1, message = "Partition count should be greater than zero.") int getPartitionCount() {
|
||||
return this.producerProperties.getPartitionCount();
|
||||
}
|
||||
|
||||
public void setPartitionCount(int partitionCount) {
|
||||
this.producerProperties.setPartitionCount(partitionCount);
|
||||
}
|
||||
|
||||
public String[] getRequiredGroups() {
|
||||
return this.producerProperties.getRequiredGroups();
|
||||
}
|
||||
|
||||
public void setRequiredGroups(String... requiredGroups) {
|
||||
this.producerProperties.setRequiredGroups(requiredGroups);
|
||||
}
|
||||
|
||||
public @AssertTrue(message = "Partition key expression and partition key extractor class properties "
|
||||
+ "are mutually exclusive.") boolean isValidPartitionKeyProperty() {
|
||||
return this.producerProperties.isValidPartitionKeyProperty();
|
||||
}
|
||||
|
||||
public @AssertTrue(message = "Partition selector class and partition selector expression "
|
||||
+ "properties are mutually exclusive.") boolean isValidPartitionSelectorProperty() {
|
||||
return this.producerProperties.isValidPartitionSelectorProperty();
|
||||
}
|
||||
|
||||
public HeaderMode getHeaderMode() {
|
||||
return this.producerProperties.getHeaderMode();
|
||||
}
|
||||
|
||||
public void setHeaderMode(HeaderMode headerMode) {
|
||||
this.producerProperties.setHeaderMode(headerMode);
|
||||
}
|
||||
|
||||
public boolean isUseNativeEncoding() {
|
||||
return this.producerProperties.isUseNativeEncoding();
|
||||
}
|
||||
|
||||
public void setUseNativeEncoding(boolean useNativeEncoding) {
|
||||
this.producerProperties.setUseNativeEncoding(useNativeEncoding);
|
||||
}
|
||||
|
||||
public boolean isErrorChannelEnabled() {
|
||||
return this.producerProperties.isErrorChannelEnabled();
|
||||
}
|
||||
|
||||
public void setErrorChannelEnabled(boolean errorChannelEnabled) {
|
||||
this.producerProperties.setErrorChannelEnabled(errorChannelEnabled);
|
||||
}
|
||||
|
||||
public String getPartitionKeyExtractorName() {
|
||||
return this.producerProperties.getPartitionKeyExtractorName();
|
||||
}
|
||||
|
||||
public void setPartitionKeyExtractorName(String partitionKeyExtractorName) {
|
||||
this.producerProperties
|
||||
.setPartitionKeyExtractorName(partitionKeyExtractorName);
|
||||
}
|
||||
|
||||
public String getPartitionSelectorName() {
|
||||
return this.producerProperties.getPartitionSelectorName();
|
||||
}
|
||||
|
||||
public void setPartitionSelectorName(String partitionSelectorName) {
|
||||
this.producerProperties.setPartitionSelectorName(partitionSelectorName);
|
||||
}
|
||||
|
||||
public int getBufferSize() {
|
||||
return this.kafkaProducerProperties.getBufferSize();
|
||||
}
|
||||
|
||||
public void setBufferSize(int bufferSize) {
|
||||
this.kafkaProducerProperties.setBufferSize(bufferSize);
|
||||
}
|
||||
|
||||
public @NotNull CompressionType getCompressionType() {
|
||||
return this.kafkaProducerProperties.getCompressionType();
|
||||
}
|
||||
|
||||
public void setCompressionType(CompressionType compressionType) {
|
||||
this.kafkaProducerProperties.setCompressionType(compressionType);
|
||||
}
|
||||
|
||||
public boolean isSync() {
|
||||
return this.kafkaProducerProperties.isSync();
|
||||
}
|
||||
|
||||
public void setSync(boolean sync) {
|
||||
this.kafkaProducerProperties.setSync(sync);
|
||||
}
|
||||
|
||||
public int getBatchTimeout() {
|
||||
return this.kafkaProducerProperties.getBatchTimeout();
|
||||
}
|
||||
|
||||
public void setBatchTimeout(int batchTimeout) {
|
||||
this.kafkaProducerProperties.setBatchTimeout(batchTimeout);
|
||||
}
|
||||
|
||||
public Expression getMessageKeyExpression() {
|
||||
return this.kafkaProducerProperties.getMessageKeyExpression();
|
||||
}
|
||||
|
||||
public void setMessageKeyExpression(Expression messageKeyExpression) {
|
||||
this.kafkaProducerProperties.setMessageKeyExpression(messageKeyExpression);
|
||||
}
|
||||
|
||||
public String[] getHeaderPatterns() {
|
||||
return this.kafkaProducerProperties.getHeaderPatterns();
|
||||
}
|
||||
|
||||
public void setHeaderPatterns(String[] headerPatterns) {
|
||||
this.kafkaProducerProperties.setHeaderPatterns(headerPatterns);
|
||||
}
|
||||
|
||||
public Map<String, String> getConfiguration() {
|
||||
return this.kafkaProducerProperties.getConfiguration();
|
||||
}
|
||||
|
||||
public void setConfiguration(Map<String, String> configuration) {
|
||||
this.kafkaProducerProperties.setConfiguration(configuration);
|
||||
}
|
||||
|
||||
public KafkaTopicProperties getTopic() {
|
||||
return this.kafkaProducerProperties.getTopic();
|
||||
}
|
||||
|
||||
public void setTopic(KafkaTopicProperties topic) {
|
||||
this.kafkaProducerProperties.setTopic(topic);
|
||||
}
|
||||
|
||||
public KafkaProducerProperties getExtension() {
|
||||
return this.kafkaProducerProperties;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class Metrics {
|
||||
/**
|
||||
* When set to true, the offset lag metric of each consumer topic is computed whenever the metric is queried (default: true).
|
||||
* When set to false only the periodically calculated offset lag is used.
|
||||
* See {@link #offsetLagMetricsInterval} for more information.
|
||||
*/
|
||||
private boolean defaultOffsetLagMetricsEnabled = true;
|
||||
|
||||
/**
|
||||
* The interval in which the offset lag for each consumer topic is computed (default: 60 seconds).
|
||||
* This value is used whenever {@link #defaultOffsetLagMetricsEnabled} is disabled or its computation is taking too long.
|
||||
*/
|
||||
private Duration offsetLagMetricsInterval = Duration.ofSeconds(60);
|
||||
|
||||
public boolean isDefaultOffsetLagMetricsEnabled() {
|
||||
return defaultOffsetLagMetricsEnabled;
|
||||
}
|
||||
|
||||
public void setDefaultOffsetLagMetricsEnabled(boolean defaultOffsetLagMetricsEnabled) {
|
||||
this.defaultOffsetLagMetricsEnabled = defaultOffsetLagMetricsEnabled;
|
||||
}
|
||||
|
||||
public Duration getOffsetLagMetricsInterval() {
|
||||
return offsetLagMetricsInterval;
|
||||
}
|
||||
|
||||
public void setOffsetLagMetricsInterval(Duration offsetLagMetricsInterval) {
|
||||
this.offsetLagMetricsInterval = offsetLagMetricsInterval;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
/*
|
||||
* Copyright 2016-2018 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.properties;
|
||||
|
||||
import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider;
|
||||
|
||||
/**
|
||||
* Container object for Kafka specific extended producer and consumer binding properties.
|
||||
*
|
||||
* @author Marius Bogoevici
|
||||
* @author Oleg Zhurakousky
|
||||
*/
|
||||
public class KafkaBindingProperties implements BinderSpecificPropertiesProvider {
|
||||
|
||||
/**
|
||||
* Consumer specific binding properties. @see {@link KafkaConsumerProperties}.
|
||||
*/
|
||||
private KafkaConsumerProperties consumer = new KafkaConsumerProperties();
|
||||
|
||||
/**
|
||||
* Producer specific binding properties. @see {@link KafkaProducerProperties}.
|
||||
*/
|
||||
private KafkaProducerProperties producer = new KafkaProducerProperties();
|
||||
|
||||
/**
|
||||
* @return {@link KafkaConsumerProperties}
|
||||
* Consumer specific binding properties. @see {@link KafkaConsumerProperties}.
|
||||
*/
|
||||
public KafkaConsumerProperties getConsumer() {
|
||||
return this.consumer;
|
||||
}
|
||||
|
||||
public void setConsumer(KafkaConsumerProperties consumer) {
|
||||
this.consumer = consumer;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return {@link KafkaProducerProperties}
|
||||
* Producer specific binding properties. @see {@link KafkaProducerProperties}.
|
||||
*/
|
||||
public KafkaProducerProperties getProducer() {
|
||||
return this.producer;
|
||||
}
|
||||
|
||||
public void setProducer(KafkaProducerProperties producer) {
|
||||
this.producer = producer;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,576 +0,0 @@
|
||||
/*
|
||||
* Copyright 2016-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.properties;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.kafka.listener.ContainerProperties;
|
||||
|
||||
/**
|
||||
* Extended consumer properties for Kafka binder.
|
||||
*
|
||||
* @author Marius Bogoevici
|
||||
* @author Ilayaperumal Gopinathan
|
||||
* @author Soby Chacko
|
||||
* @author Gary Russell
|
||||
* @author Aldo Sinanaj
|
||||
*
|
||||
* <p>
|
||||
* Thanks to Laszlo Szabo for providing the initial patch for generic property support.
|
||||
* </p>
|
||||
*/
|
||||
public class KafkaConsumerProperties {
|
||||
|
||||
/**
|
||||
* Enumeration for starting consumer offset.
|
||||
*/
|
||||
public enum StartOffset {
|
||||
|
||||
/**
|
||||
* Starting from earliest offset.
|
||||
*/
|
||||
earliest(-2L),
|
||||
/**
|
||||
* Starting from latest offset.
|
||||
*/
|
||||
latest(-1L);
|
||||
|
||||
private final long referencePoint;
|
||||
|
||||
StartOffset(long referencePoint) {
|
||||
this.referencePoint = referencePoint;
|
||||
}
|
||||
|
||||
public long getReferencePoint() {
|
||||
return this.referencePoint;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Standard headers for the message.
|
||||
*/
|
||||
public enum StandardHeaders {
|
||||
|
||||
/**
|
||||
* No headers.
|
||||
*/
|
||||
none,
|
||||
/**
|
||||
* Message header representing ID.
|
||||
*/
|
||||
id,
|
||||
/**
|
||||
* Message header representing timestamp.
|
||||
*/
|
||||
timestamp,
|
||||
/**
|
||||
* Indicating both ID and timestamp headers.
|
||||
*/
|
||||
both
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* When true the offset is committed after each record, otherwise the offsets for the complete set of records
|
||||
* received from the poll() are committed after all records have been processed.
|
||||
*/
|
||||
@Deprecated
|
||||
private boolean ackEachRecord;
|
||||
|
||||
/**
|
||||
* When true, topic partitions is automatically rebalanced between the members of a consumer group.
|
||||
* When false, each consumer is assigned a fixed set of partitions based on spring.cloud.stream.instanceCount and spring.cloud.stream.instanceIndex.
|
||||
*/
|
||||
private boolean autoRebalanceEnabled = true;
|
||||
|
||||
/**
|
||||
* Whether to autocommit offsets when a message has been processed.
|
||||
* If set to false, a header with the key kafka_acknowledgment of the type org.springframework.kafka.support.Acknowledgment header
|
||||
* is present in the inbound message. Applications may use this header for acknowledging messages.
|
||||
*/
|
||||
@Deprecated
|
||||
private boolean autoCommitOffset = true;
|
||||
|
||||
/**
|
||||
* Controlling the container acknowledgement mode. This is the preferred way to control the ack mode on the
|
||||
* container instead of the deprecated autoCommitOffset property.
|
||||
*/
|
||||
private ContainerProperties.AckMode ackMode;
|
||||
|
||||
/**
|
||||
* Flag to enable auto commit on error in polled consumers.
|
||||
*/
|
||||
private Boolean autoCommitOnError;
|
||||
|
||||
/**
|
||||
* The starting offset for new groups. Allowed values: earliest and latest.
|
||||
*/
|
||||
private StartOffset startOffset;
|
||||
|
||||
/**
|
||||
* Whether to reset offsets on the consumer to the value provided by startOffset.
|
||||
* Must be false if a KafkaRebalanceListener is provided.
|
||||
*/
|
||||
private boolean resetOffsets;
|
||||
|
||||
/**
|
||||
* When set to true, it enables DLQ behavior for the consumer.
|
||||
* By default, messages that result in errors are forwarded to a topic named error.name-of-destination.name-of-group.
|
||||
* The DLQ topic name can be configurable by setting the dlqName property.
|
||||
*/
|
||||
private boolean enableDlq;
|
||||
|
||||
/**
|
||||
* The name of the DLQ topic to receive the error messages.
|
||||
*/
|
||||
private String dlqName;
|
||||
|
||||
/**
|
||||
* Number of partitions to use on the DLQ.
|
||||
*/
|
||||
private Integer dlqPartitions;
|
||||
|
||||
/**
|
||||
* Using this, DLQ-specific producer properties can be set.
|
||||
* All the properties available through kafka producer properties can be set through this property.
|
||||
*/
|
||||
private KafkaProducerProperties dlqProducerProperties = new KafkaProducerProperties();
|
||||
|
||||
/**
|
||||
* @deprecated No longer used by the binder.
|
||||
*/
|
||||
@Deprecated
|
||||
private int recoveryInterval = 5000;
|
||||
|
||||
/**
|
||||
* List of trusted packages to provide the header mapper.
|
||||
*/
|
||||
private String[] trustedPackages;
|
||||
|
||||
/**
|
||||
* Indicates which standard headers are populated by the inbound channel adapter.
|
||||
* Allowed values: none, id, timestamp, or both.
|
||||
*/
|
||||
private StandardHeaders standardHeaders = StandardHeaders.none;
|
||||
|
||||
/**
|
||||
* The name of a bean that implements RecordMessageConverter.
|
||||
*/
|
||||
private String converterBeanName;
|
||||
|
||||
/**
|
||||
* The interval, in milliseconds, between events indicating that no messages have recently been received.
|
||||
*/
|
||||
private long idleEventInterval = 30_000;
|
||||
|
||||
/**
|
||||
* When true, the destination is treated as a regular expression Pattern used to match topic names by the broker.
|
||||
*/
|
||||
private boolean destinationIsPattern;
|
||||
|
||||
/**
|
||||
* Map with a key/value pair containing generic Kafka consumer properties.
|
||||
* In addition to having Kafka consumer properties, other configuration properties can be passed here.
|
||||
*/
|
||||
private Map<String, String> configuration = new HashMap<>();
|
||||
|
||||
/**
|
||||
* Various topic level properties. @see {@link KafkaTopicProperties} for more details.
|
||||
*/
|
||||
private KafkaTopicProperties topic = new KafkaTopicProperties();
|
||||
|
||||
/**
|
||||
* Timeout used for polling in pollable consumers.
|
||||
*/
|
||||
private long pollTimeout = org.springframework.kafka.listener.ConsumerProperties.DEFAULT_POLL_TIMEOUT;
|
||||
|
||||
/**
|
||||
* Transaction manager bean name - overrides the binder's transaction configuration.
|
||||
*/
|
||||
private String transactionManager;
|
||||
|
||||
/**
|
||||
* Set to false to NOT commit the offset of a successfully recovered recovered in the after rollback processor.
|
||||
*/
|
||||
private boolean txCommitRecovered = true;
|
||||
|
||||
/**
|
||||
* CommonErrorHandler bean name per consumer binding.
|
||||
* @since 3.2
|
||||
*/
|
||||
private String commonErrorHandlerBeanName;
|
||||
|
||||
/**
|
||||
* When using the reactive binder, automatically commit offsets when records received
|
||||
* by the poll have been processed.
|
||||
* @since 4.0.2
|
||||
*/
|
||||
private boolean reactiveAutoCommit;
|
||||
|
||||
/**
|
||||
* When using the reactive binder, automatically commit offsets of records before they
|
||||
* are passed to the user code.
|
||||
* @since 4.0.3
|
||||
*/
|
||||
private boolean reactiveAtMostOnce;
|
||||
|
||||
/**
|
||||
* @return if each record needs to be acknowledged.
|
||||
*
|
||||
* When true the offset is committed after each record, otherwise the offsets for the complete set of records
|
||||
* received from the poll() are committed after all records have been processed.
|
||||
*
|
||||
* @deprecated since 3.1 in favor of using {@link #ackMode}
|
||||
*/
|
||||
@Deprecated
|
||||
public boolean isAckEachRecord() {
|
||||
return this.ackEachRecord;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ackEachRecord
|
||||
*
|
||||
* @deprecated in favor of using {@link #ackMode}
|
||||
*/
|
||||
@Deprecated
|
||||
public void setAckEachRecord(boolean ackEachRecord) {
|
||||
this.ackEachRecord = ackEachRecord;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return is autocommit offset enabled
|
||||
*
|
||||
* Whether to autocommit offsets when a message has been processed.
|
||||
* If set to false, a header with the key kafka_acknowledgment of the type org.springframework.kafka.support.Acknowledgment header
|
||||
* is present in the inbound message. Applications may use this header for acknowledging messages.
|
||||
*
|
||||
* @deprecated since 3.1 in favor of using {@link #ackMode}
|
||||
*/
|
||||
@Deprecated
|
||||
public boolean isAutoCommitOffset() {
|
||||
return this.autoCommitOffset;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param autoCommitOffset
|
||||
*
|
||||
* @deprecated in favor of using {@link #ackMode}
|
||||
*/
|
||||
@Deprecated
|
||||
public void setAutoCommitOffset(boolean autoCommitOffset) {
|
||||
this.autoCommitOffset = autoCommitOffset;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Container's ack mode.
|
||||
*/
|
||||
public ContainerProperties.AckMode getAckMode() {
|
||||
return this.ackMode;
|
||||
}
|
||||
|
||||
public void setAckMode(ContainerProperties.AckMode ackMode) {
|
||||
this.ackMode = ackMode;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return start offset
|
||||
*
|
||||
* The starting offset for new groups. Allowed values: earliest and latest.
|
||||
*/
|
||||
public StartOffset getStartOffset() {
|
||||
return this.startOffset;
|
||||
}
|
||||
|
||||
public void setStartOffset(StartOffset startOffset) {
|
||||
this.startOffset = startOffset;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return if resetting offset is enabled
|
||||
*
|
||||
* Whether to reset offsets on the consumer to the value provided by startOffset.
|
||||
* Must be false if a KafkaRebalanceListener is provided.
|
||||
*/
|
||||
public boolean isResetOffsets() {
|
||||
return this.resetOffsets;
|
||||
}
|
||||
|
||||
public void setResetOffsets(boolean resetOffsets) {
|
||||
this.resetOffsets = resetOffsets;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return is DLQ enabled.
|
||||
*
|
||||
* When set to true, it enables DLQ behavior for the consumer.
|
||||
* By default, messages that result in errors are forwarded to a topic named error.name-of-destination.name-of-group.
|
||||
* The DLQ topic name can be configurable by setting the dlqName property.
|
||||
*/
|
||||
public boolean isEnableDlq() {
|
||||
return this.enableDlq;
|
||||
}
|
||||
|
||||
public void setEnableDlq(boolean enableDlq) {
|
||||
this.enableDlq = enableDlq;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return is autocommit on error in polled consumers.
|
||||
*
|
||||
* This property accessor is only used in polled consumers.
|
||||
*/
|
||||
public Boolean getAutoCommitOnError() {
|
||||
return this.autoCommitOnError;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param autoCommitOnError commit on error in polled consumers.
|
||||
*
|
||||
*/
|
||||
public void setAutoCommitOnError(Boolean autoCommitOnError) {
|
||||
this.autoCommitOnError = autoCommitOnError;
|
||||
}
|
||||
|
||||
/**
|
||||
* No longer used.
|
||||
* @return the interval.
|
||||
* @deprecated No longer used by the binder
|
||||
*/
|
||||
@Deprecated
|
||||
public int getRecoveryInterval() {
|
||||
return this.recoveryInterval;
|
||||
}
|
||||
|
||||
/**
|
||||
* No longer used.
|
||||
* @param recoveryInterval the interval.
|
||||
* @deprecated No longer needed by the binder
|
||||
*/
|
||||
@Deprecated
|
||||
public void setRecoveryInterval(int recoveryInterval) {
|
||||
this.recoveryInterval = recoveryInterval;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return is auto rebalance enabled
|
||||
*
|
||||
* When true, topic partitions is automatically rebalanced between the members of a consumer group.
|
||||
* When false, each consumer is assigned a fixed set of partitions based on spring.cloud.stream.instanceCount and spring.cloud.stream.instanceIndex.
|
||||
*/
|
||||
public boolean isAutoRebalanceEnabled() {
|
||||
return this.autoRebalanceEnabled;
|
||||
}
|
||||
|
||||
public void setAutoRebalanceEnabled(boolean autoRebalanceEnabled) {
|
||||
this.autoRebalanceEnabled = autoRebalanceEnabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return a map of configuration
|
||||
*
|
||||
* Map with a key/value pair containing generic Kafka consumer properties.
|
||||
* In addition to having Kafka consumer properties, other configuration properties can be passed here.
|
||||
*/
|
||||
public Map<String, String> getConfiguration() {
|
||||
return this.configuration;
|
||||
}
|
||||
|
||||
public void setConfiguration(Map<String, String> configuration) {
|
||||
this.configuration = configuration;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return dlq name
|
||||
*
|
||||
* The name of the DLQ topic to receive the error messages.
|
||||
*/
|
||||
public String getDlqName() {
|
||||
return this.dlqName;
|
||||
}
|
||||
|
||||
public void setDlqName(String dlqName) {
|
||||
this.dlqName = dlqName;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return number of partitions on the DLQ topic
|
||||
*
|
||||
* Number of partitions to use on the DLQ.
|
||||
*/
|
||||
public Integer getDlqPartitions() {
|
||||
return this.dlqPartitions;
|
||||
}
|
||||
|
||||
public void setDlqPartitions(Integer dlqPartitions) {
|
||||
this.dlqPartitions = dlqPartitions;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return trusted packages
|
||||
*
|
||||
* List of trusted packages to provide the header mapper.
|
||||
*/
|
||||
public String[] getTrustedPackages() {
|
||||
return this.trustedPackages;
|
||||
}
|
||||
|
||||
public void setTrustedPackages(String[] trustedPackages) {
|
||||
this.trustedPackages = trustedPackages;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return dlq producer properties
|
||||
*
|
||||
* Using this, DLQ-specific producer properties can be set.
|
||||
* All the properties available through kafka producer properties can be set through this property.
|
||||
*/
|
||||
public KafkaProducerProperties getDlqProducerProperties() {
|
||||
return this.dlqProducerProperties;
|
||||
}
|
||||
|
||||
public void setDlqProducerProperties(KafkaProducerProperties dlqProducerProperties) {
|
||||
this.dlqProducerProperties = dlqProducerProperties;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return standard headers
|
||||
*
|
||||
* Indicates which standard headers are populated by the inbound channel adapter.
|
||||
* Allowed values: none, id, timestamp, or both.
|
||||
*/
|
||||
public StandardHeaders getStandardHeaders() {
|
||||
return this.standardHeaders;
|
||||
}
|
||||
|
||||
public void setStandardHeaders(StandardHeaders standardHeaders) {
|
||||
this.standardHeaders = standardHeaders;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return converter bean name
|
||||
*
|
||||
* The name of a bean that implements RecordMessageConverter.
|
||||
*/
|
||||
public String getConverterBeanName() {
|
||||
return this.converterBeanName;
|
||||
}
|
||||
|
||||
public void setConverterBeanName(String converterBeanName) {
|
||||
this.converterBeanName = converterBeanName;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return idle event interval
|
||||
*
|
||||
* The interval, in milliseconds, between events indicating that no messages have recently been received.
|
||||
*/
|
||||
public long getIdleEventInterval() {
|
||||
return this.idleEventInterval;
|
||||
}
|
||||
|
||||
public void setIdleEventInterval(long idleEventInterval) {
|
||||
this.idleEventInterval = idleEventInterval;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return is destination given through a pattern
|
||||
*
|
||||
* When true, the destination is treated as a regular expression Pattern used to match topic names by the broker.
|
||||
*/
|
||||
public boolean isDestinationIsPattern() {
|
||||
return this.destinationIsPattern;
|
||||
}
|
||||
|
||||
public void setDestinationIsPattern(boolean destinationIsPattern) {
|
||||
this.destinationIsPattern = destinationIsPattern;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return topic properties
|
||||
*
|
||||
* Various topic level properties. @see {@link KafkaTopicProperties} for more details.
|
||||
*/
|
||||
public KafkaTopicProperties getTopic() {
|
||||
return this.topic;
|
||||
}
|
||||
|
||||
public void setTopic(KafkaTopicProperties topic) {
|
||||
this.topic = topic;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return timeout in pollable consumers
|
||||
*
|
||||
* Timeout used for polling in pollable consumers.
|
||||
*/
|
||||
public long getPollTimeout() {
|
||||
return this.pollTimeout;
|
||||
}
|
||||
|
||||
public void setPollTimeout(long pollTimeout) {
|
||||
this.pollTimeout = pollTimeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the transaction manager bean name.
|
||||
*
|
||||
* Transaction manager bean name (must be {@code KafkaAwareTransactionManager}.
|
||||
*/
|
||||
public String getTransactionManager() {
|
||||
return this.transactionManager;
|
||||
}
|
||||
|
||||
public void setTransactionManager(String transactionManager) {
|
||||
this.transactionManager = transactionManager;
|
||||
}
|
||||
|
||||
public boolean isTxCommitRecovered() {
|
||||
return this.txCommitRecovered;
|
||||
}
|
||||
|
||||
public void setTxCommitRecovered(boolean txCommitRecovered) {
|
||||
this.txCommitRecovered = txCommitRecovered;
|
||||
}
|
||||
|
||||
public String getCommonErrorHandlerBeanName() {
|
||||
return commonErrorHandlerBeanName;
|
||||
}
|
||||
|
||||
public void setCommonErrorHandlerBeanName(String commonErrorHandlerBeanName) {
|
||||
this.commonErrorHandlerBeanName = commonErrorHandlerBeanName;
|
||||
}
|
||||
|
||||
public boolean isReactiveAutoCommit() {
|
||||
return this.reactiveAutoCommit;
|
||||
}
|
||||
|
||||
public void setReactiveAutoCommit(boolean reactiveAutoCommit) {
|
||||
this.reactiveAutoCommit = reactiveAutoCommit;
|
||||
}
|
||||
|
||||
public boolean isReactiveAtMostOnce() {
|
||||
return this.reactiveAtMostOnce;
|
||||
}
|
||||
|
||||
public void setReactiveAtMostOnce(boolean reactiveAtMostOnce) {
|
||||
this.reactiveAtMostOnce = reactiveAtMostOnce;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
/*
|
||||
* Copyright 2016-2018 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.properties;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.AbstractExtendedBindingProperties;
|
||||
import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider;
|
||||
|
||||
/**
|
||||
* Kafka specific extended binding properties class that extends from
|
||||
* {@link AbstractExtendedBindingProperties}.
|
||||
*
|
||||
* @author Marius Bogoevici
|
||||
* @author Gary Russell
|
||||
* @author Soby Chacko
|
||||
* @author Oleg Zhurakousky
|
||||
*/
|
||||
@ConfigurationProperties("spring.cloud.stream.kafka")
|
||||
public class KafkaExtendedBindingProperties extends
|
||||
AbstractExtendedBindingProperties<KafkaConsumerProperties, KafkaProducerProperties, KafkaBindingProperties> {
|
||||
|
||||
private static final String DEFAULTS_PREFIX = "spring.cloud.stream.kafka.default";
|
||||
|
||||
@Override
|
||||
public String getDefaultsPrefix() {
|
||||
return DEFAULTS_PREFIX;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, KafkaBindingProperties> getBindings() {
|
||||
return this.doGetBindings();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<? extends BinderSpecificPropertiesProvider> getExtendedPropertiesEntryClass() {
|
||||
return KafkaBindingProperties.class;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,328 +0,0 @@
|
||||
/*
|
||||
* Copyright 2016-2018 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.properties;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
|
||||
import org.springframework.expression.Expression;
|
||||
|
||||
/**
|
||||
* Extended producer properties for Kafka binder.
|
||||
*
|
||||
* @author Marius Bogoevici
|
||||
* @author Henryk Konsek
|
||||
* @author Gary Russell
|
||||
* @author Aldo Sinanaj
|
||||
*/
|
||||
public class KafkaProducerProperties {
|
||||
|
||||
/**
|
||||
* Upper limit, in bytes, of how much data the Kafka producer attempts to batch before sending.
|
||||
*/
|
||||
private int bufferSize = 16384;
|
||||
|
||||
/**
|
||||
* Set the compression.type producer property. Supported values are none, gzip, snappy and lz4.
|
||||
* See {@link CompressionType} for more details.
|
||||
*/
|
||||
private CompressionType compressionType = CompressionType.none;
|
||||
|
||||
/**
|
||||
* Whether the producer is synchronous.
|
||||
*/
|
||||
private boolean sync;
|
||||
|
||||
/**
|
||||
* A SpEL expression evaluated against the outgoing message used to evaluate the time to wait
|
||||
* for ack when synchronous publish is enabled.
|
||||
*/
|
||||
private Expression sendTimeoutExpression;
|
||||
|
||||
/**
|
||||
* How long the producer waits to allow more messages to accumulate in the same batch before sending the messages.
|
||||
*/
|
||||
private int batchTimeout;
|
||||
|
||||
/**
|
||||
* A SpEL expression evaluated against the outgoing message used to populate the key of the produced Kafka message.
|
||||
*/
|
||||
private Expression messageKeyExpression;
|
||||
|
||||
/**
|
||||
* A comma-delimited list of simple patterns to match Spring messaging headers
|
||||
* to be mapped to the Kafka Headers in the ProducerRecord.
|
||||
*/
|
||||
private String[] headerPatterns;
|
||||
|
||||
/**
|
||||
* Map with a key/value pair containing generic Kafka producer properties.
|
||||
*/
|
||||
private Map<String, String> configuration = new HashMap<>();
|
||||
|
||||
/**
|
||||
* Various topic level properties. @see {@link KafkaTopicProperties} for more details.
|
||||
*/
|
||||
private KafkaTopicProperties topic = new KafkaTopicProperties();
|
||||
|
||||
/**
|
||||
* Set to true to override the default binding destination (topic name) with the value of the
|
||||
* KafkaHeaders.TOPIC message header in the outbound message. If the header is not present,
|
||||
* the default binding destination is used.
|
||||
*/
|
||||
private boolean useTopicHeader;
|
||||
|
||||
/**
|
||||
* The bean name of a MessageChannel to which successful send results should be sent;
|
||||
* the bean must exist in the application context. When using the reactive binder,
|
||||
* the channel must be of type FluxMessageChannel.
|
||||
*/
|
||||
private String recordMetadataChannel;
|
||||
|
||||
/**
|
||||
* Transaction manager bean name - overrides the binder's transaction configuration.
|
||||
*/
|
||||
private String transactionManager;
|
||||
|
||||
/*
|
||||
* Timeout value in seconds for the duration to wait when closing the producer.
|
||||
* If not set this defaults to 30 seconds.
|
||||
*/
|
||||
private int closeTimeout;
|
||||
|
||||
/**
|
||||
* Set to true to disable transactions.
|
||||
*/
|
||||
private boolean allowNonTransactional;
|
||||
|
||||
/**
|
||||
* @return buffer size
|
||||
*
|
||||
* Upper limit, in bytes, of how much data the Kafka producer attempts to batch before sending.
|
||||
*/
|
||||
public int getBufferSize() {
|
||||
return this.bufferSize;
|
||||
}
|
||||
|
||||
public void setBufferSize(int bufferSize) {
|
||||
this.bufferSize = bufferSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return compression type {@link CompressionType}
|
||||
*
|
||||
* Set the compression.type producer property. Supported values are none, gzip, snappy, lz4 and zstd.
|
||||
* See {@link CompressionType} for more details.
|
||||
*/
|
||||
@NotNull
|
||||
public CompressionType getCompressionType() {
|
||||
return this.compressionType;
|
||||
}
|
||||
|
||||
public void setCompressionType(CompressionType compressionType) {
|
||||
this.compressionType = compressionType;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return if synchronous sending is enabled
|
||||
*
|
||||
* Whether the producer is synchronous.
|
||||
*/
|
||||
public boolean isSync() {
|
||||
return this.sync;
|
||||
}
|
||||
|
||||
public void setSync(boolean sync) {
|
||||
this.sync = sync;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return timeout expression for send
|
||||
*
|
||||
* A SpEL expression evaluated against the outgoing message used to evaluate the time to wait
|
||||
* for ack when synchronous publish is enabled.
|
||||
*/
|
||||
public Expression getSendTimeoutExpression() {
|
||||
return this.sendTimeoutExpression;
|
||||
}
|
||||
|
||||
public void setSendTimeoutExpression(Expression sendTimeoutExpression) {
|
||||
this.sendTimeoutExpression = sendTimeoutExpression;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return batch timeout
|
||||
*
|
||||
* How long the producer waits to allow more messages to accumulate in the same batch before sending the messages.
|
||||
*/
|
||||
public int getBatchTimeout() {
|
||||
return this.batchTimeout;
|
||||
}
|
||||
|
||||
public void setBatchTimeout(int batchTimeout) {
|
||||
this.batchTimeout = batchTimeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return message key expression
|
||||
*
|
||||
* A SpEL expression evaluated against the outgoing message used to populate the key of the produced Kafka message.
|
||||
*/
|
||||
public Expression getMessageKeyExpression() {
|
||||
return this.messageKeyExpression;
|
||||
}
|
||||
|
||||
public void setMessageKeyExpression(Expression messageKeyExpression) {
|
||||
this.messageKeyExpression = messageKeyExpression;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return header patterns
|
||||
*
|
||||
* A comma-delimited list of simple patterns to match Spring messaging headers
|
||||
* to be mapped to the Kafka Headers in the ProducerRecord.
|
||||
*/
|
||||
public String[] getHeaderPatterns() {
|
||||
return this.headerPatterns;
|
||||
}
|
||||
|
||||
public void setHeaderPatterns(String[] headerPatterns) {
|
||||
this.headerPatterns = headerPatterns;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return map of configuration
|
||||
*
|
||||
* Map with a key/value pair containing generic Kafka producer properties.
|
||||
*/
|
||||
public Map<String, String> getConfiguration() {
|
||||
return this.configuration;
|
||||
}
|
||||
|
||||
public void setConfiguration(Map<String, String> configuration) {
|
||||
this.configuration = configuration;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return topic properties
|
||||
*
|
||||
* Various topic level properties. @see {@link KafkaTopicProperties} for more details.
|
||||
*/
|
||||
public KafkaTopicProperties getTopic() {
|
||||
return this.topic;
|
||||
}
|
||||
|
||||
public void setTopic(KafkaTopicProperties topic) {
|
||||
this.topic = topic;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return if using topic header
|
||||
*
|
||||
* Set to true to override the default binding destination (topic name) with the value of the
|
||||
* KafkaHeaders.TOPIC message header in the outbound message. If the header is not present,
|
||||
* the default binding destination is used.
|
||||
*/
|
||||
public boolean isUseTopicHeader() {
|
||||
return this.useTopicHeader;
|
||||
}
|
||||
|
||||
public void setUseTopicHeader(boolean useTopicHeader) {
|
||||
this.useTopicHeader = useTopicHeader;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return record metadata channel
|
||||
*
|
||||
* The bean name of a MessageChannel to which successful send results should be sent;
|
||||
* the bean must exist in the application context.
|
||||
*/
|
||||
public String getRecordMetadataChannel() {
|
||||
return this.recordMetadataChannel;
|
||||
}
|
||||
|
||||
public void setRecordMetadataChannel(String recordMetadataChannel) {
|
||||
this.recordMetadataChannel = recordMetadataChannel;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the transaction manager bean name.
|
||||
*
|
||||
* Transaction manager bean name (must be {@code KafkaAwareTransactionManager}.
|
||||
*/
|
||||
public String getTransactionManager() {
|
||||
return this.transactionManager;
|
||||
}
|
||||
|
||||
public void setTransactionManager(String transactionManager) {
|
||||
this.transactionManager = transactionManager;
|
||||
}
|
||||
|
||||
/*
|
||||
* @return timeout in seconds for closing the producer
|
||||
*/
|
||||
public int getCloseTimeout() {
|
||||
return this.closeTimeout;
|
||||
}
|
||||
|
||||
public void setCloseTimeout(int closeTimeout) {
|
||||
this.closeTimeout = closeTimeout;
|
||||
}
|
||||
|
||||
public boolean isAllowNonTransactional() {
|
||||
return this.allowNonTransactional;
|
||||
}
|
||||
|
||||
public void setAllowNonTransactional(boolean allowNonTransactional) {
|
||||
this.allowNonTransactional = allowNonTransactional;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enumeration for compression types.
|
||||
*/
|
||||
public enum CompressionType {
|
||||
|
||||
/**
|
||||
* No compression.
|
||||
*/
|
||||
none,
|
||||
|
||||
/**
|
||||
* gzip based compression.
|
||||
*/
|
||||
gzip,
|
||||
|
||||
/**
|
||||
* snappy based compression.
|
||||
*/
|
||||
snappy,
|
||||
|
||||
/**
|
||||
* lz4 compression.
|
||||
*/
|
||||
lz4,
|
||||
|
||||
/**
|
||||
* zstd compression.
|
||||
*/
|
||||
zstd,
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
/*
|
||||
* Copyright 2019-2019 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.properties;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Properties for configuring topics.
|
||||
*
|
||||
* @author Aldo Sinanaj
|
||||
* @since 2.2
|
||||
*
|
||||
*/
|
||||
public class KafkaTopicProperties {
|
||||
|
||||
private Short replicationFactor;
|
||||
|
||||
private Map<Integer, List<Integer>> replicasAssignments = new HashMap<>();
|
||||
|
||||
private Map<String, String> properties = new HashMap<>();
|
||||
|
||||
public Short getReplicationFactor() {
|
||||
return replicationFactor;
|
||||
}
|
||||
|
||||
public void setReplicationFactor(Short replicationFactor) {
|
||||
this.replicationFactor = replicationFactor;
|
||||
}
|
||||
|
||||
public Map<Integer, List<Integer>> getReplicasAssignments() {
|
||||
return replicasAssignments;
|
||||
}
|
||||
|
||||
public void setReplicasAssignments(Map<Integer, List<Integer>> replicasAssignments) {
|
||||
this.replicasAssignments = replicasAssignments;
|
||||
}
|
||||
|
||||
public Map<String, String> getProperties() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
public void setProperties(Map<String, String> properties) {
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021-2021 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.provisioning;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Customizer for configuring AdminClient.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @since 3.1.2
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface AdminClientConfigCustomizer {
|
||||
|
||||
void configure(Map<String, Object> adminClientProperties);
|
||||
}
|
||||
@@ -1,690 +0,0 @@
|
||||
/*
|
||||
* Copyright 2014-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.provisioning;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.kafka.clients.CommonClientConfigs;
|
||||
import org.apache.kafka.clients.admin.AdminClient;
|
||||
import org.apache.kafka.clients.admin.AdminClientConfig;
|
||||
import org.apache.kafka.clients.admin.AlterConfigOp;
|
||||
import org.apache.kafka.clients.admin.AlterConfigsResult;
|
||||
import org.apache.kafka.clients.admin.Config;
|
||||
import org.apache.kafka.clients.admin.ConfigEntry;
|
||||
import org.apache.kafka.clients.admin.CreatePartitionsResult;
|
||||
import org.apache.kafka.clients.admin.CreateTopicsResult;
|
||||
import org.apache.kafka.clients.admin.DescribeConfigsResult;
|
||||
import org.apache.kafka.clients.admin.DescribeTopicsResult;
|
||||
import org.apache.kafka.clients.admin.ListTopicsResult;
|
||||
import org.apache.kafka.clients.admin.NewPartitions;
|
||||
import org.apache.kafka.clients.admin.NewTopic;
|
||||
import org.apache.kafka.clients.admin.TopicDescription;
|
||||
import org.apache.kafka.common.KafkaFuture;
|
||||
import org.apache.kafka.common.PartitionInfo;
|
||||
import org.apache.kafka.common.config.ConfigResource;
|
||||
import org.apache.kafka.common.errors.TopicExistsException;
|
||||
import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
|
||||
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
|
||||
import org.springframework.cloud.stream.binder.BinderException;
|
||||
import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaTopicProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.utils.KafkaTopicUtils;
|
||||
import org.springframework.cloud.stream.provisioning.ConsumerDestination;
|
||||
import org.springframework.cloud.stream.provisioning.ProducerDestination;
|
||||
import org.springframework.cloud.stream.provisioning.ProvisioningException;
|
||||
import org.springframework.cloud.stream.provisioning.ProvisioningProvider;
|
||||
import org.springframework.retry.RetryOperations;
|
||||
import org.springframework.retry.backoff.ExponentialBackOffPolicy;
|
||||
import org.springframework.retry.policy.SimpleRetryPolicy;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.ObjectUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* Kafka implementation for {@link ProvisioningProvider}.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @author Gary Russell
|
||||
* @author Ilayaperumal Gopinathan
|
||||
* @author Simon Flandergan
|
||||
* @author Oleg Zhurakousky
|
||||
* @author Aldo Sinanaj
|
||||
* @author Yi Liu
|
||||
* @author Omer Celik
|
||||
* @author Byungjun You
|
||||
*/
|
||||
public class KafkaTopicProvisioner implements
|
||||
// @checkstyle:off
|
||||
ProvisioningProvider<ExtendedConsumerProperties<KafkaConsumerProperties>, ExtendedProducerProperties<KafkaProducerProperties>>,
|
||||
// @checkstyle:on
|
||||
InitializingBean {
|
||||
|
||||
private static final Log logger = LogFactory.getLog(KafkaTopicProvisioner.class);
|
||||
|
||||
private static final int DEFAULT_OPERATION_TIMEOUT = 30;
|
||||
|
||||
private final KafkaBinderConfigurationProperties configurationProperties;
|
||||
|
||||
private final int operationTimeout = DEFAULT_OPERATION_TIMEOUT;
|
||||
|
||||
private final Map<String, Object> adminClientProperties;
|
||||
|
||||
private RetryOperations metadataRetryOperations;
|
||||
|
||||
/**
|
||||
* Create an instance.
|
||||
* @param kafkaBinderConfigurationProperties the binder configuration properties.
|
||||
* @param kafkaProperties the boot Kafka properties used to build the
|
||||
* @param adminClientConfigCustomizer to customize {@link AdminClient}.
|
||||
* {@link AdminClient}.
|
||||
*/
|
||||
public KafkaTopicProvisioner(
|
||||
KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties,
|
||||
KafkaProperties kafkaProperties,
|
||||
AdminClientConfigCustomizer adminClientConfigCustomizer) {
|
||||
this(kafkaBinderConfigurationProperties, kafkaProperties, adminClientConfigCustomizer != null ?
|
||||
Arrays.asList(adminClientConfigCustomizer) : new ArrayList<>());
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an instance.
|
||||
*
|
||||
* @param kafkaBinderConfigurationProperties the binder configuration properties.
|
||||
* @param kafkaProperties the boot Kafka properties used to build the
|
||||
* @param adminClientConfigCustomizers to customize {@link AdminClient}.
|
||||
* {@link AdminClient}.
|
||||
*/
|
||||
public KafkaTopicProvisioner(
|
||||
KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties,
|
||||
KafkaProperties kafkaProperties,
|
||||
List<AdminClientConfigCustomizer> adminClientConfigCustomizers) {
|
||||
|
||||
Assert.isTrue(kafkaProperties != null, "KafkaProperties cannot be null");
|
||||
this.configurationProperties = kafkaBinderConfigurationProperties;
|
||||
this.adminClientProperties = kafkaProperties.buildAdminProperties();
|
||||
normalalizeBootPropsWithBinder(this.adminClientProperties, kafkaProperties,
|
||||
kafkaBinderConfigurationProperties);
|
||||
// If the application provides AdminConfig customizers
|
||||
// and overrides properties, those take precedence.
|
||||
adminClientConfigCustomizers.forEach(customizer -> customizer.configure(this.adminClientProperties));
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an unmodifiable map of merged admin properties.
|
||||
* @return the properties.
|
||||
* @since 4.0.3
|
||||
*/
|
||||
public Map<String, Object> getAdminClientProperties() {
|
||||
return Collections.unmodifiableMap(this.adminClientProperties);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mutator for metadata retry operations.
|
||||
* @param metadataRetryOperations the retry configuration
|
||||
*/
|
||||
public void setMetadataRetryOperations(RetryOperations metadataRetryOperations) {
|
||||
this.metadataRetryOperations = metadataRetryOperations;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void afterPropertiesSet() {
|
||||
if (this.metadataRetryOperations == null) {
|
||||
RetryTemplate retryTemplate = new RetryTemplate();
|
||||
|
||||
SimpleRetryPolicy simpleRetryPolicy = new SimpleRetryPolicy();
|
||||
simpleRetryPolicy.setMaxAttempts(10);
|
||||
retryTemplate.setRetryPolicy(simpleRetryPolicy);
|
||||
|
||||
ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
|
||||
backOffPolicy.setInitialInterval(100);
|
||||
backOffPolicy.setMultiplier(2);
|
||||
backOffPolicy.setMaxInterval(1000);
|
||||
retryTemplate.setBackOffPolicy(backOffPolicy);
|
||||
this.metadataRetryOperations = retryTemplate;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProducerDestination provisionProducerDestination(final String name,
|
||||
ExtendedProducerProperties<KafkaProducerProperties> properties) {
|
||||
|
||||
if (logger.isInfoEnabled()) {
|
||||
logger.info("Using kafka topic for outbound: " + name);
|
||||
}
|
||||
if (this.configurationProperties.isAutoCreateTopics()) {
|
||||
KafkaTopicUtils.validateTopicName(name);
|
||||
try (AdminClient adminClient = createAdminClient()) {
|
||||
createTopic(adminClient, name, properties.getPartitionCount(), false,
|
||||
properties.getExtension().getTopic());
|
||||
int partitions = getPartitionsForTopic(name, adminClient);
|
||||
return new KafkaProducerDestination(name, partitions);
|
||||
}
|
||||
}
|
||||
return new KafkaProducerDestination(name, 0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConsumerDestination provisionConsumerDestination(final String name,
|
||||
final String group,
|
||||
ExtendedConsumerProperties<KafkaConsumerProperties> properties) {
|
||||
if (!properties.isMultiplex()) {
|
||||
return doProvisionConsumerDestination(name, group, properties);
|
||||
}
|
||||
else {
|
||||
String[] destinations = StringUtils.commaDelimitedListToStringArray(name);
|
||||
for (String destination : destinations) {
|
||||
doProvisionConsumerDestination(destination.trim(), group, properties);
|
||||
}
|
||||
return new KafkaConsumerDestination(name);
|
||||
}
|
||||
}
|
||||
|
||||
private ConsumerDestination doProvisionConsumerDestination(final String name,
|
||||
final String group,
|
||||
ExtendedConsumerProperties<KafkaConsumerProperties> properties) {
|
||||
final KafkaConsumerDestination kafkaConsumerDestination = new KafkaConsumerDestination(name);
|
||||
if (properties.getExtension().isDestinationIsPattern()) {
|
||||
Assert.isTrue(!properties.getExtension().isEnableDlq(),
|
||||
"enableDLQ is not allowed when listening to topic patterns");
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Listening to a topic pattern - " + name
|
||||
+ " - no provisioning performed");
|
||||
}
|
||||
return kafkaConsumerDestination;
|
||||
}
|
||||
if (this.configurationProperties.isAutoCreateTopics()) {
|
||||
KafkaTopicUtils.validateTopicName(name);
|
||||
boolean anonymous = !StringUtils.hasText(group);
|
||||
Assert.isTrue(!anonymous || !properties.getExtension().isEnableDlq(),
|
||||
"DLQ support is not available for anonymous subscriptions");
|
||||
if (properties.getInstanceCount() == 0) {
|
||||
throw new IllegalArgumentException("Instance count cannot be zero");
|
||||
}
|
||||
int partitionCount = properties.getInstanceCount() * properties.getConcurrency();
|
||||
ConsumerDestination consumerDestination;
|
||||
try (AdminClient adminClient = createAdminClient()) {
|
||||
createTopic(adminClient, name, partitionCount,
|
||||
properties.getExtension().isAutoRebalanceEnabled(),
|
||||
properties.getExtension().getTopic());
|
||||
int partitions = getPartitionsForTopic(name, adminClient);
|
||||
consumerDestination = createDlqIfNeedBe(adminClient, name, group,
|
||||
properties, anonymous, partitions);
|
||||
if (consumerDestination == null) {
|
||||
consumerDestination = new KafkaConsumerDestination(name,
|
||||
partitions);
|
||||
}
|
||||
return consumerDestination;
|
||||
}
|
||||
}
|
||||
return kafkaConsumerDestination;
|
||||
}
|
||||
private int getPartitionsForTopic(String topicName, AdminClient adminClient) {
|
||||
int partitions = 0;
|
||||
Map<String, TopicDescription> topicDescriptions = retrieveTopicDescriptions(topicName, adminClient);
|
||||
TopicDescription topicDescription = topicDescriptions.get(topicName);
|
||||
if (topicDescription != null) {
|
||||
partitions = topicDescription.partitions().size();
|
||||
}
|
||||
return partitions;
|
||||
}
|
||||
|
||||
private Map<String, TopicDescription> retrieveTopicDescriptions(String topicName, AdminClient adminClient) {
|
||||
return this.metadataRetryOperations.execute(context -> {
|
||||
try {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Attempting to retrieve the description for the topic: " + topicName);
|
||||
}
|
||||
DescribeTopicsResult describeTopicsResult = adminClient
|
||||
.describeTopics(Collections.singletonList(topicName));
|
||||
KafkaFuture<Map<String, TopicDescription>> all = describeTopicsResult
|
||||
.allTopicNames();
|
||||
return all.get(this.operationTimeout, TimeUnit.SECONDS);
|
||||
}
|
||||
catch (Exception ex) {
|
||||
throw new ProvisioningException("Problems encountered with partitions finding for: " + topicName, ex);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
AdminClient createAdminClient() {
|
||||
return AdminClient.create(this.adminClientProperties);
|
||||
}
|
||||
|
||||
/**
|
||||
* In general, binder properties supersede boot kafka properties. The one exception is
|
||||
* the bootstrap servers. In that case, we should only override the boot properties if
|
||||
* (there is a binder property AND it is a non-default value) OR (if there is no boot
|
||||
* property); this is needed because the binder property never returns a null value.
|
||||
* @param adminProps the admin properties to normalize.
|
||||
* @param bootProps the boot kafka properties.
|
||||
* @param binderProps the binder kafka properties.
|
||||
*/
|
||||
public static void normalalizeBootPropsWithBinder(Map<String, Object> adminProps,
|
||||
KafkaProperties bootProps, KafkaBinderConfigurationProperties binderProps) {
|
||||
// First deal with the outlier
|
||||
String kafkaConnectionString = binderProps.getKafkaConnectionString();
|
||||
if (ObjectUtils
|
||||
.isEmpty(adminProps.get(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG))
|
||||
|| !kafkaConnectionString
|
||||
.equals(binderProps.getDefaultKafkaConnectionString())) {
|
||||
adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG,
|
||||
kafkaConnectionString);
|
||||
}
|
||||
// Now override any boot values with binder values
|
||||
Map<String, String> binderProperties = binderProps.getConfiguration();
|
||||
Set<String> adminConfigNames = AdminClientConfig.configNames();
|
||||
binderProperties.forEach((key, value) -> {
|
||||
if (key.equals(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG)) {
|
||||
throw new IllegalStateException(
|
||||
"Set binder bootstrap servers via the 'brokers' property, not 'configuration'");
|
||||
}
|
||||
if (adminConfigNames.contains(key)) {
|
||||
Object replaced = adminProps.put(key, value);
|
||||
if (replaced != null && KafkaTopicProvisioner.logger.isDebugEnabled()) {
|
||||
KafkaTopicProvisioner.logger.debug("Overrode boot property: [" + key + "], from: ["
|
||||
+ replaced + "] to: [" + value + "]");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private ConsumerDestination createDlqIfNeedBe(AdminClient adminClient, String name,
|
||||
String group, ExtendedConsumerProperties<KafkaConsumerProperties> properties,
|
||||
boolean anonymous, int partitions) {
|
||||
|
||||
if (properties.getExtension().isEnableDlq() && !anonymous) {
|
||||
String dlqTopic = StringUtils.hasText(properties.getExtension().getDlqName())
|
||||
? properties.getExtension().getDlqName()
|
||||
: "error." + name + "." + group;
|
||||
int dlqPartitions = properties.getExtension().getDlqPartitions() == null
|
||||
? partitions
|
||||
: properties.getExtension().getDlqPartitions();
|
||||
try {
|
||||
final KafkaProducerProperties dlqProducerProperties = properties.getExtension().getDlqProducerProperties();
|
||||
createTopicAndPartitions(adminClient, dlqTopic, dlqPartitions,
|
||||
properties.getExtension().isAutoRebalanceEnabled(),
|
||||
dlqProducerProperties.getTopic());
|
||||
}
|
||||
catch (Throwable throwable) {
|
||||
if (throwable instanceof Error throwableError) {
|
||||
throw throwableError;
|
||||
}
|
||||
else {
|
||||
throw new ProvisioningException("Provisioning exception encountered for " + name, throwable);
|
||||
}
|
||||
}
|
||||
return new KafkaConsumerDestination(name, partitions, dlqTopic);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void createTopic(AdminClient adminClient, String name, int partitionCount,
|
||||
boolean tolerateLowerPartitionsOnBroker, KafkaTopicProperties properties) {
|
||||
try {
|
||||
createTopicIfNecessary(adminClient, name, partitionCount,
|
||||
tolerateLowerPartitionsOnBroker, properties);
|
||||
}
|
||||
// TODO: Remove catching Throwable. See this thread:
|
||||
// TODO:
|
||||
// https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/pull/514#discussion_r241075940
|
||||
catch (Throwable throwable) {
|
||||
if (throwable instanceof Error throwableError) {
|
||||
throw throwableError;
|
||||
}
|
||||
else {
|
||||
// TODO:
|
||||
// https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/pull/514#discussion_r241075940
|
||||
throw new ProvisioningException("Provisioning exception encountered for " + name, throwable);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void createTopicIfNecessary(AdminClient adminClient, final String topicName,
|
||||
final int partitionCount, boolean tolerateLowerPartitionsOnBroker,
|
||||
KafkaTopicProperties properties) throws Throwable {
|
||||
|
||||
if (this.configurationProperties.isAutoCreateTopics()) {
|
||||
createTopicAndPartitions(adminClient, topicName, partitionCount,
|
||||
tolerateLowerPartitionsOnBroker, properties);
|
||||
}
|
||||
else {
|
||||
logger.info("Auto creation of topics is disabled.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a Kafka topic if needed, or try to increase its partition count to the
|
||||
* desired number.
|
||||
* @param adminClient kafka admin client
|
||||
* @param topicName topic name
|
||||
* @param partitionCount partition count
|
||||
* @param tolerateLowerPartitionsOnBroker whether lower partitions count on broker is
|
||||
* tolerated ot not
|
||||
* @param topicProperties kafka topic properties
|
||||
* @throws Throwable from topic creation
|
||||
*/
|
||||
private void createTopicAndPartitions(AdminClient adminClient, final String topicName,
|
||||
final int partitionCount, boolean tolerateLowerPartitionsOnBroker,
|
||||
KafkaTopicProperties topicProperties) throws Throwable {
|
||||
|
||||
ListTopicsResult listTopicsResult = adminClient.listTopics();
|
||||
KafkaFuture<Set<String>> namesFutures = listTopicsResult.names();
|
||||
|
||||
Set<String> names = namesFutures.get(this.operationTimeout, TimeUnit.SECONDS);
|
||||
if (names.contains(topicName)) {
|
||||
//check if topic.properties are different from Topic Configuration in Kafka
|
||||
if (this.configurationProperties.isAutoAlterTopics()) {
|
||||
alterTopicConfigsIfNecessary(adminClient, topicName, topicProperties);
|
||||
}
|
||||
// only consider minPartitionCount for resizing if autoAddPartitions is true
|
||||
int effectivePartitionCount = this.configurationProperties
|
||||
.isAutoAddPartitions()
|
||||
? Math.max(
|
||||
this.configurationProperties.getMinPartitionCount(),
|
||||
partitionCount)
|
||||
: partitionCount;
|
||||
DescribeTopicsResult describeTopicsResult = adminClient
|
||||
.describeTopics(Collections.singletonList(topicName));
|
||||
KafkaFuture<Map<String, TopicDescription>> topicDescriptionsFuture = describeTopicsResult
|
||||
.all();
|
||||
Map<String, TopicDescription> topicDescriptions = topicDescriptionsFuture
|
||||
.get(this.operationTimeout, TimeUnit.SECONDS);
|
||||
TopicDescription topicDescription = topicDescriptions.get(topicName);
|
||||
int partitionSize = topicDescription.partitions().size();
|
||||
if (partitionSize < effectivePartitionCount) {
|
||||
if (this.configurationProperties.isAutoAddPartitions()) {
|
||||
CreatePartitionsResult partitions = adminClient
|
||||
.createPartitions(Collections.singletonMap(topicName,
|
||||
NewPartitions.increaseTo(effectivePartitionCount)));
|
||||
partitions.all().get(this.operationTimeout, TimeUnit.SECONDS);
|
||||
}
|
||||
else if (tolerateLowerPartitionsOnBroker) {
|
||||
logger.warn("The number of expected partitions for topic "
|
||||
+ topicName + " was: "
|
||||
+ partitionCount + ", but " + partitionSize
|
||||
+ (partitionSize > 1 ? " have " : " has ")
|
||||
+ "been found instead. " + "There will be "
|
||||
+ (effectivePartitionCount - partitionSize)
|
||||
+ " idle consumers");
|
||||
}
|
||||
else {
|
||||
throw new ProvisioningException(
|
||||
"The number of expected partitions for topic " + topicName
|
||||
+ " was: " + partitionCount
|
||||
+ ", but " + partitionSize
|
||||
+ (partitionSize > 1 ? " have " : " has ")
|
||||
+ "been found instead. "
|
||||
+ "Consider either increasing the partition count of the topic or enabling "
|
||||
+ "`autoAddPartitions`");
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// always consider minPartitionCount for topic creation
|
||||
final int effectivePartitionCount = Math.max(
|
||||
this.configurationProperties.getMinPartitionCount(), partitionCount);
|
||||
this.metadataRetryOperations.execute((context) -> {
|
||||
|
||||
NewTopic newTopic;
|
||||
Map<Integer, List<Integer>> replicasAssignments = topicProperties
|
||||
.getReplicasAssignments();
|
||||
if (replicasAssignments != null && replicasAssignments.size() > 0) {
|
||||
newTopic = new NewTopic(topicName,
|
||||
topicProperties.getReplicasAssignments());
|
||||
}
|
||||
else {
|
||||
newTopic = new NewTopic(topicName, effectivePartitionCount,
|
||||
topicProperties.getReplicationFactor() != null
|
||||
? topicProperties.getReplicationFactor()
|
||||
: this.configurationProperties
|
||||
.getReplicationFactor());
|
||||
}
|
||||
if (topicProperties.getProperties().size() > 0) {
|
||||
newTopic.configs(topicProperties.getProperties());
|
||||
}
|
||||
CreateTopicsResult createTopicsResult = adminClient
|
||||
.createTopics(Collections.singletonList(newTopic));
|
||||
try {
|
||||
createTopicsResult.all().get(this.operationTimeout, TimeUnit.SECONDS);
|
||||
}
|
||||
catch (Exception ex) {
|
||||
if (ex instanceof ExecutionException) {
|
||||
if (ex.getCause() instanceof TopicExistsException) {
|
||||
if (logger.isWarnEnabled()) {
|
||||
logger.warn("Attempt to create topic: " + topicName
|
||||
+ ". Topic already exists.");
|
||||
}
|
||||
}
|
||||
else {
|
||||
logger.error("Failed to create topics", ex.getCause());
|
||||
throw ex.getCause();
|
||||
}
|
||||
}
|
||||
else {
|
||||
logger.error("Failed to create topics", ex.getCause());
|
||||
throw ex.getCause();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private void alterTopicConfigsIfNecessary(AdminClient adminClient,
|
||||
String topicName,
|
||||
KafkaTopicProperties topicProperties)
|
||||
throws InterruptedException, ExecutionException, java.util.concurrent.TimeoutException {
|
||||
ConfigResource topicConfigResource = new ConfigResource(ConfigResource.Type.TOPIC, topicName);
|
||||
DescribeConfigsResult describeConfigsResult = adminClient
|
||||
.describeConfigs(Collections.singletonList(topicConfigResource));
|
||||
KafkaFuture<Map<ConfigResource, Config>> topicConfigurationFuture = describeConfigsResult.all();
|
||||
Map<ConfigResource, Config> topicConfigMap = topicConfigurationFuture
|
||||
.get(this.operationTimeout, TimeUnit.SECONDS);
|
||||
Config config = topicConfigMap.get(topicConfigResource);
|
||||
final List<AlterConfigOp> updatedConfigEntries = topicProperties.getProperties().entrySet().stream()
|
||||
.filter(propertiesEntry -> {
|
||||
// Property is new and should be added
|
||||
if (config.get(propertiesEntry.getKey()) == null) {
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
// Property changed and should be updated
|
||||
return !config.get(propertiesEntry.getKey()).value().equals(propertiesEntry.getValue());
|
||||
}
|
||||
|
||||
})
|
||||
.map(propertyEntry -> new ConfigEntry(propertyEntry.getKey(), propertyEntry.getValue()))
|
||||
.map(configEntry -> new AlterConfigOp(configEntry, AlterConfigOp.OpType.SET))
|
||||
.collect(Collectors.toList());
|
||||
if (!updatedConfigEntries.isEmpty()) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Attempting to alter configs " + updatedConfigEntries + " for the topic:" + topicName);
|
||||
}
|
||||
Map<ConfigResource, Collection<AlterConfigOp>> alterConfigForTopics = new HashMap<>();
|
||||
alterConfigForTopics.put(topicConfigResource, updatedConfigEntries);
|
||||
AlterConfigsResult alterConfigsResult = adminClient.incrementalAlterConfigs(alterConfigForTopics);
|
||||
alterConfigsResult.all().get(this.operationTimeout, TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check that the topic has the expected number of partitions and return the partition information.
|
||||
* @param partitionCount the expected count.
|
||||
* @param tolerateLowerPartitionsOnBroker if false, throw an exception if there are not enough partitions.
|
||||
* @param callable a Callable that will provide the partition information.
|
||||
* @param topicName the topic./
|
||||
* @return the partition information.
|
||||
*/
|
||||
public Collection<PartitionInfo> getPartitionsForTopic(final int partitionCount,
|
||||
final boolean tolerateLowerPartitionsOnBroker,
|
||||
final Callable<Collection<PartitionInfo>> callable, final String topicName) {
|
||||
try {
|
||||
return this.metadataRetryOperations.execute((context) -> {
|
||||
Collection<PartitionInfo> partitions = Collections.emptyList();
|
||||
|
||||
try {
|
||||
// This call may return null or throw an exception.
|
||||
partitions = callable.call();
|
||||
}
|
||||
catch (Exception ex) {
|
||||
// The above call can potentially throw exceptions such as timeout. If
|
||||
// we can determine
|
||||
// that the exception was due to an unknown topic on the broker, just
|
||||
// simply rethrow that.
|
||||
if (ex instanceof UnknownTopicOrPartitionException) {
|
||||
throw ex;
|
||||
}
|
||||
logger.error("Failed to obtain partition information for the topic "
|
||||
+ "(" + topicName + ").", ex);
|
||||
}
|
||||
// In some cases, the above partition query may not throw an UnknownTopic..Exception for various reasons.
|
||||
// For that, we are forcing another query to ensure that the topic is present on the server.
|
||||
if (CollectionUtils.isEmpty(partitions)) {
|
||||
try (AdminClient adminClient = createAdminClient()) {
|
||||
final DescribeTopicsResult describeTopicsResult = adminClient
|
||||
.describeTopics(Collections.singletonList(topicName));
|
||||
|
||||
describeTopicsResult.all().get();
|
||||
}
|
||||
catch (ExecutionException ex) {
|
||||
if (ex.getCause() instanceof UnknownTopicOrPartitionException unknownTopicOrPartitionException) {
|
||||
throw unknownTopicOrPartitionException;
|
||||
}
|
||||
else {
|
||||
logger.warn("No partitions have been retrieved for the topic "
|
||||
+ "(" + topicName
|
||||
+ "). This will affect the health check.");
|
||||
}
|
||||
}
|
||||
}
|
||||
// do a sanity check on the partition set
|
||||
int partitionSize = CollectionUtils.isEmpty(partitions) ? 0 : partitions.size();
|
||||
if (partitionSize < partitionCount) {
|
||||
if (tolerateLowerPartitionsOnBroker) {
|
||||
logger.warn("The number of expected partitions for topic "
|
||||
+ topicName + " was: "
|
||||
+ partitionCount + ", but " + partitionSize
|
||||
+ (partitionSize > 1 ? " have " : " has ")
|
||||
+ "been found instead. " + "There will be "
|
||||
+ (partitionCount - partitionSize) + " idle consumers");
|
||||
}
|
||||
else {
|
||||
throw new IllegalStateException(
|
||||
"The number of expected partitions for topic " + topicName
|
||||
+ " was: " + partitionCount
|
||||
+ ", but " + partitionSize
|
||||
+ (partitionSize > 1 ? " have " : " has ")
|
||||
+ "been found instead");
|
||||
}
|
||||
}
|
||||
return partitions;
|
||||
});
|
||||
}
|
||||
catch (Exception ex) {
|
||||
logger.error("Cannot initialize Binder checking the topic (" + topicName + ").", ex);
|
||||
throw new BinderException("Cannot initialize binder checking the topic (" + topicName + "):", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static final class KafkaProducerDestination implements ProducerDestination {
|
||||
|
||||
private final String producerDestinationName;
|
||||
|
||||
private final int partitions;
|
||||
|
||||
KafkaProducerDestination(String destinationName, Integer partitions) {
|
||||
this.producerDestinationName = destinationName;
|
||||
this.partitions = partitions;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return this.producerDestinationName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNameForPartition(int partition) {
|
||||
return this.producerDestinationName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "KafkaProducerDestination{" + "producerDestinationName='"
|
||||
+ producerDestinationName + '\'' + ", partitions=" + partitions + '}';
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static final class KafkaConsumerDestination implements ConsumerDestination {
|
||||
|
||||
private final String consumerDestinationName;
|
||||
|
||||
private final int partitions;
|
||||
|
||||
private final String dlqName;
|
||||
|
||||
KafkaConsumerDestination(String consumerDestinationName) {
|
||||
this(consumerDestinationName, 0, null);
|
||||
}
|
||||
|
||||
KafkaConsumerDestination(String consumerDestinationName, int partitions) {
|
||||
this(consumerDestinationName, partitions, null);
|
||||
}
|
||||
|
||||
KafkaConsumerDestination(String consumerDestinationName, Integer partitions,
|
||||
String dlqName) {
|
||||
this.consumerDestinationName = consumerDestinationName;
|
||||
this.partitions = partitions;
|
||||
this.dlqName = dlqName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return this.consumerDestinationName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "KafkaConsumerDestination{" + "consumerDestinationName='"
|
||||
+ consumerDestinationName + '\'' + ", partitions=" + partitions
|
||||
+ ", dlqName='" + dlqName + '\'' + '}';
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.support;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* This customizer is called by the binder to customize consumer configuration in
|
||||
* Kafka Consumer factory.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @since 3.0.9
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface ConsumerConfigCustomizer {
|
||||
|
||||
void configure(Map<String, Object> consumerProperties, String bindingName, String destination);
|
||||
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.support;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* This customizer is called by the binder to customize producer configuration in
|
||||
* Kafka Producer factory.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @since 3.0.9
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface ProducerConfigCustomizer {
|
||||
|
||||
void configure(Map<String, Object> producerProperties, String bindingName, String destination);
|
||||
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.utils;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.kafka.clients.consumer.ConsumerConfig;
|
||||
import org.apache.kafka.clients.producer.ProducerConfig;
|
||||
import org.apache.kafka.common.serialization.ByteArrayDeserializer;
|
||||
import org.apache.kafka.common.serialization.ByteArraySerializer;
|
||||
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.beans.factory.NoSuchBeanDefinitionException;
|
||||
import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties.StandardHeaders;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.kafka.support.DefaultKafkaHeaderMapper;
|
||||
import org.springframework.kafka.support.KafkaHeaderMapper;
|
||||
import org.springframework.kafka.support.converter.MessageConverter;
|
||||
import org.springframework.kafka.support.converter.MessagingMessageConverter;
|
||||
import org.springframework.lang.Nullable;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.ObjectUtils;
|
||||
|
||||
/**
|
||||
* Binding Utilities.
|
||||
*
|
||||
* @author Gary Russell
|
||||
* @since 4.0
|
||||
*
|
||||
*/
|
||||
public final class BindingUtils {
|
||||
|
||||
private BindingUtils() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the message converter for consumer bindings from the application context. If
|
||||
* the binding properties do not contain a bean name, a default
|
||||
* {@link MessagingMessageConverter} is returned; if the binder properties contain a
|
||||
* header mapper bean name, it is used in the default converter, otherwise a
|
||||
* {@link DefaultKafkaHeaderMapper} is used.
|
||||
* @param applicationContext the application context.
|
||||
* @param extendedConsumerProperties the consumer binding properties.
|
||||
* @param configurationProperties the binder properties.
|
||||
* @return the converter
|
||||
* @throws IllegalStateException if a bean name is specified but not found.
|
||||
*/
|
||||
public static MessageConverter getConsumerMessageConverter(ApplicationContext applicationContext,
|
||||
ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties,
|
||||
KafkaBinderConfigurationProperties configurationProperties) {
|
||||
|
||||
MessageConverter messageConverter;
|
||||
if (extendedConsumerProperties.getExtension().getConverterBeanName() == null) {
|
||||
MessagingMessageConverter mmc = new MessagingMessageConverter();
|
||||
StandardHeaders standardHeaders = extendedConsumerProperties.getExtension()
|
||||
.getStandardHeaders();
|
||||
mmc.setGenerateMessageId(StandardHeaders.id.equals(standardHeaders)
|
||||
|| StandardHeaders.both.equals(standardHeaders));
|
||||
mmc.setGenerateTimestamp(
|
||||
StandardHeaders.timestamp.equals(standardHeaders)
|
||||
|| StandardHeaders.both.equals(standardHeaders));
|
||||
KafkaHeaderMapper headerMapper = getHeaderMapper(applicationContext, configurationProperties);
|
||||
if (headerMapper == null) {
|
||||
headerMapper = new DefaultKafkaHeaderMapper();
|
||||
}
|
||||
mmc.setHeaderMapper(headerMapper);
|
||||
messageConverter = mmc;
|
||||
}
|
||||
else {
|
||||
try {
|
||||
messageConverter = applicationContext.getBean(
|
||||
extendedConsumerProperties.getExtension().getConverterBeanName(), MessageConverter.class);
|
||||
}
|
||||
catch (NoSuchBeanDefinitionException ex) {
|
||||
throw new IllegalStateException(
|
||||
"Converter bean not present in application context", ex);
|
||||
}
|
||||
}
|
||||
return messageConverter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the header mapper bean, if the binder properties contains a bean name; if not
|
||||
* look for a bean with name {@code kafkaBinderHeaderMapper} is looked up; if that
|
||||
* doesn't exist, null is returned.
|
||||
* @param applicationContext the application context.
|
||||
* @param configurationProperties the binder properties.
|
||||
* @return the mapper.
|
||||
*/
|
||||
@Nullable
|
||||
public static KafkaHeaderMapper getHeaderMapper(ApplicationContext applicationContext,
|
||||
KafkaBinderConfigurationProperties configurationProperties) {
|
||||
|
||||
KafkaHeaderMapper mapper = null;
|
||||
if (configurationProperties.getHeaderMapperBeanName() != null) {
|
||||
mapper = applicationContext.getBean(
|
||||
configurationProperties.getHeaderMapperBeanName(),
|
||||
KafkaHeaderMapper.class);
|
||||
}
|
||||
if (mapper == null) {
|
||||
//First, try to see if there is a bean named headerMapper registered by other frameworks using the binder (for e.g. spring cloud sleuth)
|
||||
try {
|
||||
mapper = applicationContext.getBean("kafkaBinderHeaderMapper", KafkaHeaderMapper.class);
|
||||
}
|
||||
catch (BeansException be) {
|
||||
}
|
||||
}
|
||||
return mapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the Kafka configuration map for a consumer binding. With anonymous bindings
|
||||
* (those without a {@code group} property, which are given a {@code UUID.toString()}
|
||||
* in the group id) consumption begins from the current end of the topic, otherwise
|
||||
* consumption starts from the beginning, the first time the binding consumes.
|
||||
* @param anonymous true if this is for an anonymous binding.
|
||||
* @param consumerGroup the group.
|
||||
* @param consumerProperties the binding properties.
|
||||
* @param configurationProperties the binder properties.
|
||||
* @return the config map.
|
||||
*/
|
||||
public static Map<String, Object> createConsumerConfigs(boolean anonymous, String consumerGroup,
|
||||
ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties,
|
||||
KafkaBinderConfigurationProperties configurationProperties) {
|
||||
|
||||
Map<String, Object> props = new HashMap<>();
|
||||
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
|
||||
ByteArrayDeserializer.class);
|
||||
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
|
||||
ByteArrayDeserializer.class);
|
||||
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
|
||||
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 100);
|
||||
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,
|
||||
anonymous ? "latest" : "earliest");
|
||||
props.put(ConsumerConfig.GROUP_ID_CONFIG, consumerGroup);
|
||||
|
||||
Map<String, Object> mergedConfig = configurationProperties.mergedConsumerConfiguration();
|
||||
if (!ObjectUtils.isEmpty(mergedConfig)) {
|
||||
props.putAll(mergedConfig);
|
||||
}
|
||||
if (ObjectUtils.isEmpty(props.get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG))) {
|
||||
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
|
||||
configurationProperties.getKafkaConnectionString());
|
||||
}
|
||||
Map<String, String> config = consumerProperties.getExtension().getConfiguration();
|
||||
if (!ObjectUtils.isEmpty(config)) {
|
||||
Assert.state(!config.containsKey(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG),
|
||||
ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG + " cannot be overridden at the binding level; "
|
||||
+ "use multiple binders instead");
|
||||
props.putAll(config);
|
||||
}
|
||||
if (!ObjectUtils.isEmpty(consumerProperties.getExtension().getStartOffset())) {
|
||||
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,
|
||||
consumerProperties.getExtension().getStartOffset().name());
|
||||
}
|
||||
return props;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the Kafka configuration map for a producer binding.
|
||||
* @param producerProperties the binding properties.
|
||||
* @param configurationProperties the binder properties.
|
||||
* @return the config map.
|
||||
*/
|
||||
public static Map<String, Object> createProducerConfigs(
|
||||
ExtendedProducerProperties<KafkaProducerProperties> producerProperties,
|
||||
KafkaBinderConfigurationProperties configurationProperties) {
|
||||
|
||||
Map<String, Object> props = new HashMap<>();
|
||||
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class);
|
||||
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
|
||||
ByteArraySerializer.class);
|
||||
props.put(ProducerConfig.ACKS_CONFIG,
|
||||
String.valueOf(configurationProperties.getRequiredAcks()));
|
||||
Map<String, Object> mergedConfig = configurationProperties
|
||||
.mergedProducerConfiguration();
|
||||
if (!ObjectUtils.isEmpty(mergedConfig)) {
|
||||
props.putAll(mergedConfig);
|
||||
}
|
||||
if (ObjectUtils.isEmpty(props.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG))) {
|
||||
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,
|
||||
configurationProperties.getKafkaConnectionString());
|
||||
}
|
||||
final KafkaProducerProperties kafkaProducerProperties = producerProperties.getExtension();
|
||||
if (ObjectUtils.isEmpty(props.get(ProducerConfig.BATCH_SIZE_CONFIG))) {
|
||||
props.put(ProducerConfig.BATCH_SIZE_CONFIG,
|
||||
String.valueOf(kafkaProducerProperties.getBufferSize()));
|
||||
}
|
||||
if (ObjectUtils.isEmpty(props.get(ProducerConfig.LINGER_MS_CONFIG))) {
|
||||
props.put(ProducerConfig.LINGER_MS_CONFIG,
|
||||
String.valueOf(kafkaProducerProperties.getBatchTimeout()));
|
||||
}
|
||||
if (ObjectUtils.isEmpty(props.get(ProducerConfig.COMPRESSION_TYPE_CONFIG))) {
|
||||
props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG,
|
||||
kafkaProducerProperties.getCompressionType().toString());
|
||||
}
|
||||
Map<String, String> configs = producerProperties.getExtension().getConfiguration();
|
||||
Assert.state(!configs.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG),
|
||||
ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " cannot be overridden at the binding level; "
|
||||
+ "use multiple binders instead");
|
||||
if (!ObjectUtils.isEmpty(configs)) {
|
||||
props.putAll(configs);
|
||||
}
|
||||
if (!ObjectUtils.isEmpty(kafkaProducerProperties.getConfiguration())) {
|
||||
props.putAll(kafkaProducerProperties.getConfiguration());
|
||||
}
|
||||
return props;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.utils;
|
||||
|
||||
import java.util.function.BiFunction;
|
||||
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
|
||||
/**
|
||||
* A {@link BiFunction} extension for defining DLQ destination resolvers.
|
||||
*
|
||||
* The BiFunction takes the {@link ConsumerRecord} and the exception as inputs
|
||||
* and returns a topic name as the DLQ.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @since 3.0.9
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface DlqDestinationResolver extends BiFunction<ConsumerRecord<?, ?>, Exception, String> {
|
||||
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
/*
|
||||
* Copyright 2019-2019 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.utils;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
|
||||
import org.springframework.lang.Nullable;
|
||||
|
||||
/**
|
||||
* A TriFunction that takes a consumer group, consumer record, and throwable and returns
|
||||
* which partition to publish to the dead letter topic. Returning {@code null} means Kafka
|
||||
* will choose the partition.
|
||||
*
|
||||
* @author Gary Russell
|
||||
* @since 3.0
|
||||
*
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface DlqPartitionFunction {
|
||||
|
||||
/**
|
||||
* Returns the same partition as the original recor.
|
||||
*/
|
||||
DlqPartitionFunction ORIGINAL_PARTITION = (group, rec, ex) -> rec.partition();
|
||||
|
||||
/**
|
||||
* Returns 0.
|
||||
*/
|
||||
DlqPartitionFunction PARTITION_ZERO = (group, rec, ex) -> 0;
|
||||
|
||||
/**
|
||||
* Apply the function.
|
||||
* @param group the consumer group.
|
||||
* @param record the consumer record.
|
||||
* @param throwable the exception.
|
||||
* @return the DLQ partition, or null.
|
||||
*/
|
||||
@Nullable
|
||||
Integer apply(String group, ConsumerRecord<?, ?> record, Throwable throwable);
|
||||
|
||||
/**
|
||||
* Determine the fallback function to use based on the dlq partition count if no
|
||||
* {@link DlqPartitionFunction} bean is provided.
|
||||
* @param dlqPartitions the partition count.
|
||||
* @param logger the logger.
|
||||
* @return the fallback.
|
||||
*/
|
||||
static DlqPartitionFunction determineFallbackFunction(@Nullable Integer dlqPartitions, Log logger) {
|
||||
if (dlqPartitions == null) {
|
||||
return ORIGINAL_PARTITION;
|
||||
}
|
||||
else if (dlqPartitions > 1) {
|
||||
logger.error("'dlqPartitions' is > 1 but a custom DlqPartitionFunction bean is not provided");
|
||||
return ORIGINAL_PARTITION;
|
||||
}
|
||||
else {
|
||||
return PARTITION_ZERO;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
/*
|
||||
* Copyright 2016-2019 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.utils;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* Utility methods releated to Kafka topics.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
public final class KafkaTopicUtils {
|
||||
|
||||
private KafkaTopicUtils() {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate topic name. Allowed chars are ASCII alphanumerics, '.', '_' and '-'.
|
||||
* @param topicName name of the topic
|
||||
*/
|
||||
public static void validateTopicName(String topicName) {
|
||||
byte[] utf8 = topicName.getBytes(StandardCharsets.UTF_8);
|
||||
for (byte b : utf8) {
|
||||
if (!((b >= 'a') && (b <= 'z') || (b >= 'A') && (b <= 'Z')
|
||||
|| (b >= '0') && (b <= '9') || (b == '.') || (b == '-')
|
||||
|| (b == '_'))) {
|
||||
throw new IllegalArgumentException(
|
||||
"Topic name can only have ASCII alphanumerics, '.', '_' and '-', but was: '"
|
||||
+ topicName + "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,217 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.properties;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import org.apache.kafka.clients.consumer.ConsumerConfig;
|
||||
import org.assertj.core.util.Files;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class KafkaBinderConfigurationPropertiesTest {
|
||||
|
||||
@Test
|
||||
public void mergedConsumerConfigurationFiltersGroupIdFromKafkaProperties() {
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
kafkaProperties.getConsumer().setGroupId("group1");
|
||||
KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties =
|
||||
new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
|
||||
Map<String, Object> mergedConsumerConfiguration =
|
||||
kafkaBinderConfigurationProperties.mergedConsumerConfiguration();
|
||||
|
||||
assertThat(mergedConsumerConfiguration).doesNotContainKeys(ConsumerConfig.GROUP_ID_CONFIG);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void mergedConsumerConfigurationFiltersEnableAutoCommitFromKafkaProperties() {
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
kafkaProperties.getConsumer().setEnableAutoCommit(true);
|
||||
KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties =
|
||||
new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
|
||||
Map<String, Object> mergedConsumerConfiguration =
|
||||
kafkaBinderConfigurationProperties.mergedConsumerConfiguration();
|
||||
|
||||
assertThat(mergedConsumerConfiguration).doesNotContainKeys(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void mergedConsumerConfigurationFiltersGroupIdFromKafkaBinderConfigurationPropertiesConfiguration() {
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties =
|
||||
new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
kafkaBinderConfigurationProperties
|
||||
.setConfiguration(Collections.singletonMap(ConsumerConfig.GROUP_ID_CONFIG, "group1"));
|
||||
|
||||
Map<String, Object> mergedConsumerConfiguration = kafkaBinderConfigurationProperties.mergedConsumerConfiguration();
|
||||
|
||||
assertThat(mergedConsumerConfiguration).doesNotContainKeys(ConsumerConfig.GROUP_ID_CONFIG);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void mergedConsumerConfigurationFiltersEnableAutoCommitFromKafkaBinderConfigurationPropertiesConfiguration() {
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties =
|
||||
new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
kafkaBinderConfigurationProperties
|
||||
.setConfiguration(Collections.singletonMap(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"));
|
||||
|
||||
Map<String, Object> mergedConsumerConfiguration = kafkaBinderConfigurationProperties.mergedConsumerConfiguration();
|
||||
|
||||
assertThat(mergedConsumerConfiguration).doesNotContainKeys(ConsumerConfig.GROUP_ID_CONFIG);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void mergedConsumerConfigurationFiltersGroupIdFromKafkaBinderConfigurationPropertiesConsumerProperties() {
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties =
|
||||
new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
kafkaBinderConfigurationProperties
|
||||
.setConsumerProperties(Collections.singletonMap(ConsumerConfig.GROUP_ID_CONFIG, "group1"));
|
||||
|
||||
Map<String, Object> mergedConsumerConfiguration = kafkaBinderConfigurationProperties.mergedConsumerConfiguration();
|
||||
|
||||
assertThat(mergedConsumerConfiguration).doesNotContainKeys(ConsumerConfig.GROUP_ID_CONFIG);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void mergedConsumerConfigurationFiltersEnableAutoCommitFromKafkaBinderConfigurationPropertiesConsumerProps() {
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties =
|
||||
new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
kafkaBinderConfigurationProperties
|
||||
.setConsumerProperties(Collections.singletonMap(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"));
|
||||
|
||||
Map<String, Object> mergedConsumerConfiguration = kafkaBinderConfigurationProperties.mergedConsumerConfiguration();
|
||||
|
||||
assertThat(mergedConsumerConfiguration).doesNotContainKeys(ConsumerConfig.GROUP_ID_CONFIG);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCertificateFilesAreConvertedToAbsolutePathsFromClassPathResources() {
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties =
|
||||
new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
final Map<String, String> configuration = kafkaBinderConfigurationProperties.getConfiguration();
|
||||
configuration.put("ssl.truststore.location", "classpath:testclient.truststore");
|
||||
configuration.put("ssl.keystore.location", "classpath:testclient.keystore");
|
||||
|
||||
kafkaBinderConfigurationProperties.getKafkaConnectionString();
|
||||
|
||||
assertThat(configuration.get("ssl.truststore.location"))
|
||||
.isEqualTo(Paths.get(System.getProperty("java.io.tmpdir"), "testclient.truststore").toString());
|
||||
assertThat(configuration.get("ssl.keystore.location"))
|
||||
.isEqualTo(Paths.get(System.getProperty("java.io.tmpdir"), "testclient.keystore").toString());
|
||||
deleteTempCertFiles();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCertificateFilesAreConvertedToAbsolutePathsFromHttpResources() throws IOException {
|
||||
HttpServer server = HttpServer.create(new InetSocketAddress("localhost", 5869), 0);
|
||||
createContextWithCertFileHandler(server, "testclient.truststore");
|
||||
createContextWithCertFileHandler(server, "testclient.keystore");
|
||||
server.setExecutor(null); // creates a default executor
|
||||
server.start();
|
||||
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties =
|
||||
new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
final Map<String, String> configuration = kafkaBinderConfigurationProperties.getConfiguration();
|
||||
configuration.put("ssl.truststore.location", "http://localhost:5869/testclient.truststore");
|
||||
configuration.put("ssl.keystore.location", "http://localhost:5869/testclient.keystore");
|
||||
configuration.put("schema.registry.ssl.truststore.location", "http://localhost:5869/testclient.truststore");
|
||||
configuration.put("schema.registry.ssl.keystore.location", "http://localhost:5869/testclient.keystore");
|
||||
|
||||
kafkaBinderConfigurationProperties.getKafkaConnectionString();
|
||||
|
||||
assertThat(configuration.get("ssl.truststore.location"))
|
||||
.isEqualTo(Paths.get(System.getProperty("java.io.tmpdir"), "testclient.truststore").toString());
|
||||
assertThat(configuration.get("ssl.keystore.location"))
|
||||
.isEqualTo(Paths.get(System.getProperty("java.io.tmpdir"), "testclient.keystore").toString());
|
||||
assertThat(configuration.get("schema.registry.ssl.truststore.location"))
|
||||
.isEqualTo(Paths.get(System.getProperty("java.io.tmpdir"), "testclient.truststore").toString());
|
||||
assertThat(configuration.get("schema.registry.ssl.keystore.location"))
|
||||
.isEqualTo(Paths.get(System.getProperty("java.io.tmpdir"), "testclient.keystore").toString());
|
||||
deleteTempCertFiles();
|
||||
|
||||
server.stop(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCertificateFilesAreConvertedToGivenAbsolutePathsFromClassPathResources() {
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties =
|
||||
new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
final Map<String, String> configuration = kafkaBinderConfigurationProperties.getConfiguration();
|
||||
configuration.put("ssl.truststore.location", "classpath:testclient.truststore");
|
||||
configuration.put("ssl.keystore.location", "classpath:testclient.keystore");
|
||||
kafkaBinderConfigurationProperties.setCertificateStoreDirectory("target");
|
||||
|
||||
kafkaBinderConfigurationProperties.getKafkaConnectionString();
|
||||
|
||||
assertThat(configuration.get("ssl.truststore.location")).isEqualTo(
|
||||
Paths.get(Files.currentFolder().toString(), "target", "testclient.truststore").toString());
|
||||
assertThat(configuration.get("ssl.keystore.location")).isEqualTo(
|
||||
Paths.get(Files.currentFolder().toString(), "target", "testclient.keystore").toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCertificateFilesAreMovedForSchemaRegistryConfiguration() {
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties =
|
||||
new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
final Map<String, String> configuration = kafkaBinderConfigurationProperties.getConfiguration();
|
||||
configuration.put("schema.registry.ssl.truststore.location", "classpath:testclient.truststore");
|
||||
configuration.put("schema.registry.ssl.keystore.location", "classpath:testclient.keystore");
|
||||
kafkaBinderConfigurationProperties.setCertificateStoreDirectory("target");
|
||||
|
||||
kafkaBinderConfigurationProperties.getKafkaConnectionString();
|
||||
|
||||
assertThat(configuration.get("schema.registry.ssl.truststore.location")).isEqualTo(
|
||||
Paths.get(Files.currentFolder().toString(), "target", "testclient.truststore").toString());
|
||||
assertThat(configuration.get("schema.registry.ssl.keystore.location")).isEqualTo(
|
||||
Paths.get(Files.currentFolder().toString(), "target", "testclient.keystore").toString());
|
||||
}
|
||||
|
||||
private void createContextWithCertFileHandler(HttpServer server, String path) {
|
||||
server.createContext("/" + path, exchange -> {
|
||||
ClassPathResource ts = new ClassPathResource(path);
|
||||
byte[] response = ts.getContentAsByteArray();
|
||||
exchange.sendResponseHeaders(200, response.length);
|
||||
OutputStream os = exchange.getResponseBody();
|
||||
os.write(response);
|
||||
os.close();
|
||||
});
|
||||
}
|
||||
|
||||
private void deleteTempCertFiles() {
|
||||
Paths.get(System.getProperty("java.io.tmpdir"), "testclient.truststore").toFile().delete();
|
||||
Paths.get(System.getProperty("java.io.tmpdir"), "testclient.keystore").toFile().delete();
|
||||
}
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.provisioning;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.kafka.clients.CommonClientConfigs;
|
||||
import org.apache.kafka.clients.admin.AdminClient;
|
||||
import org.apache.kafka.clients.admin.AdminClientConfig;
|
||||
import org.apache.kafka.common.config.SslConfigs;
|
||||
import org.apache.kafka.common.network.SslChannelBuilder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.kafka.test.utils.KafkaTestUtils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.fail;
|
||||
|
||||
/**
|
||||
* @author Gary Russell
|
||||
* @since 2.0
|
||||
*
|
||||
*/
|
||||
public class KafkaTopicProvisionerTests {
|
||||
|
||||
AdminClientConfigCustomizer adminClientConfigCustomizer = adminClientProperties -> adminClientProperties.put("foo", "bar");
|
||||
|
||||
@SuppressWarnings("rawtypes")
|
||||
@Test
|
||||
public void bootPropertiesOverriddenExceptServers() throws Exception {
|
||||
KafkaProperties bootConfig = new KafkaProperties();
|
||||
bootConfig.getProperties().put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,
|
||||
"PLAINTEXT");
|
||||
bootConfig.setBootstrapServers(Collections.singletonList("localhost:1234"));
|
||||
KafkaBinderConfigurationProperties binderConfig = new KafkaBinderConfigurationProperties(
|
||||
bootConfig);
|
||||
binderConfig.getConfiguration().put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG,
|
||||
"SSL");
|
||||
ClassPathResource ts = new ClassPathResource("test.truststore.ks");
|
||||
binderConfig.getConfiguration().put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG,
|
||||
ts.getFile().getAbsolutePath());
|
||||
binderConfig.setBrokers("localhost:9092");
|
||||
KafkaTopicProvisioner provisioner = new KafkaTopicProvisioner(binderConfig,
|
||||
bootConfig, adminClientConfigCustomizer);
|
||||
AdminClient adminClient = provisioner.createAdminClient();
|
||||
assertThat(KafkaTestUtils.getPropertyValue(adminClient,
|
||||
"client.selector.channelBuilder")).isInstanceOf(SslChannelBuilder.class);
|
||||
Map configs = KafkaTestUtils.getPropertyValue(adminClient,
|
||||
"client.selector.channelBuilder.configs", Map.class);
|
||||
assertThat(
|
||||
((List) configs.get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG)).get(0))
|
||||
.isEqualTo("localhost:1234");
|
||||
assertThat(configs.get("foo")).isEqualTo("bar");
|
||||
adminClient.close();
|
||||
}
|
||||
|
||||
@SuppressWarnings("rawtypes")
|
||||
@Test
|
||||
public void bootPropertiesOverriddenIncludingServers() throws Exception {
|
||||
KafkaProperties bootConfig = new KafkaProperties();
|
||||
bootConfig.getProperties().put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,
|
||||
"PLAINTEXT");
|
||||
bootConfig.setBootstrapServers(Collections.singletonList("localhost:9092"));
|
||||
KafkaBinderConfigurationProperties binderConfig = new KafkaBinderConfigurationProperties(
|
||||
bootConfig);
|
||||
binderConfig.getConfiguration().put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG,
|
||||
"SSL");
|
||||
ClassPathResource ts = new ClassPathResource("test.truststore.ks");
|
||||
binderConfig.getConfiguration().put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG,
|
||||
ts.getFile().getAbsolutePath());
|
||||
binderConfig.setBrokers("localhost:1234");
|
||||
KafkaTopicProvisioner provisioner = new KafkaTopicProvisioner(binderConfig,
|
||||
bootConfig, adminClientConfigCustomizer);
|
||||
AdminClient adminClient = provisioner.createAdminClient();
|
||||
assertThat(KafkaTestUtils.getPropertyValue(adminClient,
|
||||
"client.selector.channelBuilder")).isInstanceOf(SslChannelBuilder.class);
|
||||
Map configs = KafkaTestUtils.getPropertyValue(adminClient,
|
||||
"client.selector.channelBuilder.configs", Map.class);
|
||||
assertThat(
|
||||
((List) configs.get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG)).get(0))
|
||||
.isEqualTo("localhost:1234");
|
||||
adminClient.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void brokersInvalid() throws Exception {
|
||||
KafkaProperties bootConfig = new KafkaProperties();
|
||||
KafkaBinderConfigurationProperties binderConfig = new KafkaBinderConfigurationProperties(
|
||||
bootConfig);
|
||||
binderConfig.getConfiguration().put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG,
|
||||
"localhost:1234");
|
||||
try {
|
||||
new KafkaTopicProvisioner(binderConfig, bootConfig, adminClientConfigCustomizer);
|
||||
fail("Expected illegal state");
|
||||
}
|
||||
catch (IllegalStateException e) {
|
||||
assertThat(e.getMessage()).isEqualTo(
|
||||
"Set binder bootstrap servers via the 'brokers' property, not 'configuration'");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
@@ -1,89 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>spring-cloud-stream-binder-kafka-reactive</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>spring-cloud-stream-binder-kafka-reactive</name>
|
||||
<description>Kafka binder implementation</description>
|
||||
|
||||
<parent>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-binder-kafka-parent</artifactId>
|
||||
<version>4.1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-binder-kafka-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-actuator</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-configuration-processor</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-autoconfigure</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.kafka</groupId>
|
||||
<artifactId>spring-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.projectreactor.kafka</groupId>
|
||||
<artifactId>reactor-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.kafka</groupId>
|
||||
<artifactId>spring-kafka-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- Following dependencies are needed to support Kafka 1.1.0 client-->
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
<classifier>test</classifier>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka_2.13</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka_2.13</artifactId>
|
||||
<classifier>test</classifier>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.awaitility</groupId>
|
||||
<artifactId>awaitility</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -1,473 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.reactorkafka;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.kafka.clients.consumer.Consumer;
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
import org.apache.kafka.clients.producer.Producer;
|
||||
import org.apache.kafka.clients.producer.ProducerRecord;
|
||||
import org.apache.kafka.common.PartitionInfo;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.kafka.receiver.KafkaReceiver;
|
||||
import reactor.kafka.receiver.ReceiverOptions;
|
||||
import reactor.kafka.receiver.ReceiverRecord;
|
||||
import reactor.kafka.sender.KafkaSender;
|
||||
import reactor.kafka.sender.SenderOptions;
|
||||
import reactor.kafka.sender.SenderRecord;
|
||||
import reactor.kafka.sender.SenderResult;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.cloud.stream.binder.AbstractMessageChannelBinder;
|
||||
import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider;
|
||||
import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder;
|
||||
import org.springframework.cloud.stream.binder.kafka.common.TopicInformation;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaExtendedBindingProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
|
||||
import org.springframework.cloud.stream.binder.kafka.support.ConsumerConfigCustomizer;
|
||||
import org.springframework.cloud.stream.binder.kafka.support.ProducerConfigCustomizer;
|
||||
import org.springframework.cloud.stream.binder.kafka.utils.BindingUtils;
|
||||
import org.springframework.cloud.stream.provisioning.ConsumerDestination;
|
||||
import org.springframework.cloud.stream.provisioning.ProducerDestination;
|
||||
import org.springframework.context.Lifecycle;
|
||||
import org.springframework.context.support.AbstractApplicationContext;
|
||||
import org.springframework.integration.IntegrationMessageHeaderAccessor;
|
||||
import org.springframework.integration.channel.FluxMessageChannel;
|
||||
import org.springframework.integration.core.MessageProducer;
|
||||
import org.springframework.integration.endpoint.MessageProducerSupport;
|
||||
import org.springframework.integration.handler.AbstractMessageHandler;
|
||||
import org.springframework.integration.support.MessageBuilder;
|
||||
import org.springframework.kafka.core.ConsumerFactory;
|
||||
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
|
||||
import org.springframework.kafka.core.DefaultKafkaProducerFactory;
|
||||
import org.springframework.kafka.support.KafkaHeaders;
|
||||
import org.springframework.kafka.support.converter.KafkaMessageHeaders;
|
||||
import org.springframework.kafka.support.converter.MessageConverter;
|
||||
import org.springframework.kafka.support.converter.MessagingMessageConverter;
|
||||
import org.springframework.kafka.support.converter.RecordMessageConverter;
|
||||
import org.springframework.lang.Nullable;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.MessageChannel;
|
||||
import org.springframework.messaging.MessageHandler;
|
||||
import org.springframework.messaging.support.GenericMessage;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* @author Gary Russell
|
||||
* @author Byungjun You
|
||||
* @since 4.0
|
||||
*
|
||||
*/
|
||||
public class ReactorKafkaBinder
|
||||
extends AbstractMessageChannelBinder<ExtendedConsumerProperties<KafkaConsumerProperties>,
|
||||
ExtendedProducerProperties<KafkaProducerProperties>, KafkaTopicProvisioner>
|
||||
implements
|
||||
ExtendedPropertiesBinder<MessageChannel, KafkaConsumerProperties, KafkaProducerProperties> {
|
||||
|
||||
private static final Log logger = LogFactory.getLog(ReactorKafkaBinder.class);
|
||||
|
||||
private final KafkaBinderConfigurationProperties configurationProperties;
|
||||
|
||||
private KafkaExtendedBindingProperties extendedBindingProperties = new KafkaExtendedBindingProperties();
|
||||
|
||||
private ConsumerConfigCustomizer consumerConfigCustomizer;
|
||||
|
||||
private ProducerConfigCustomizer producerConfigCustomizer;
|
||||
|
||||
private ReceiverOptionsCustomizer<Object, Object> receiverOptionsCustomizer = (name, opts) -> opts;
|
||||
|
||||
private SenderOptionsCustomizer<Object, Object> senderOptionsCustomizer = (name, opts) -> opts;
|
||||
|
||||
private final Map<String, TopicInformation> topicsInUse = new ConcurrentHashMap<>();
|
||||
|
||||
private final Map<String, MessageProducerSupport> messageProducers = new ConcurrentHashMap<>();
|
||||
|
||||
public ReactorKafkaBinder(KafkaBinderConfigurationProperties configurationProperties,
|
||||
KafkaTopicProvisioner provisioner) {
|
||||
|
||||
super(new String[0], provisioner, null, null);
|
||||
this.configurationProperties = configurationProperties;
|
||||
}
|
||||
|
||||
public void setConsumerConfigCustomizer(ConsumerConfigCustomizer consumerConfigCustomizer) {
|
||||
this.consumerConfigCustomizer = consumerConfigCustomizer;
|
||||
}
|
||||
|
||||
public void setProducerConfigCustomizer(ProducerConfigCustomizer producerConfigCustomizer) {
|
||||
this.producerConfigCustomizer = producerConfigCustomizer;
|
||||
}
|
||||
|
||||
@SuppressWarnings({ "rawtypes", "unchecked" })
|
||||
public void receiverOptionsCustomizers(ObjectProvider<ReceiverOptionsCustomizer> customizers) {
|
||||
if (customizers.getIfUnique() != null) {
|
||||
this.receiverOptionsCustomizer = customizers.getIfUnique();
|
||||
}
|
||||
else {
|
||||
List<ReceiverOptionsCustomizer> list = customizers.orderedStream().toList();
|
||||
ReceiverOptionsCustomizer customizer = (name, opts) -> {
|
||||
ReceiverOptions<Object, Object> last = null;
|
||||
for (ReceiverOptionsCustomizer cust: list) {
|
||||
last = (ReceiverOptions<Object, Object>) cust.apply(name, opts);
|
||||
}
|
||||
return last;
|
||||
};
|
||||
if (!list.isEmpty()) {
|
||||
this.receiverOptionsCustomizer = customizer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings({ "rawtypes", "unchecked" })
|
||||
public void senderOptionsCustomizers(ObjectProvider<SenderOptionsCustomizer> customizers) {
|
||||
if (customizers.getIfUnique() != null) {
|
||||
this.senderOptionsCustomizer = customizers.getIfUnique();
|
||||
}
|
||||
else {
|
||||
List<SenderOptionsCustomizer> list = customizers.orderedStream().toList();
|
||||
SenderOptionsCustomizer customizer = (name, opts) -> {
|
||||
SenderOptions<Object, Object> last = null;
|
||||
for (SenderOptionsCustomizer cust: list) {
|
||||
last = (SenderOptions<Object, Object>) cust.apply(name, opts);
|
||||
}
|
||||
return last;
|
||||
};
|
||||
if (!list.isEmpty()) {
|
||||
this.senderOptionsCustomizer = customizer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected MessageHandler createProducerMessageHandler(ProducerDestination destination,
|
||||
ExtendedProducerProperties<KafkaProducerProperties> producerProperties, MessageChannel errorChannel)
|
||||
throws Exception {
|
||||
|
||||
Map<String, Object> configs = BindingUtils.createProducerConfigs(producerProperties,
|
||||
this.configurationProperties);
|
||||
if (this.producerConfigCustomizer != null) {
|
||||
this.producerConfigCustomizer.configure(configs, producerProperties.getBindingName(),
|
||||
destination.getName());
|
||||
}
|
||||
Map<String, Object> props = BindingUtils.createProducerConfigs(producerProperties,
|
||||
this.configurationProperties);
|
||||
DefaultKafkaProducerFactory<byte[], byte[]> producerFactory = new DefaultKafkaProducerFactory<>(props);
|
||||
Collection<PartitionInfo> partitions = provisioningProvider.getPartitionsForTopic(
|
||||
producerProperties.getPartitionCount(), false, () -> {
|
||||
Producer<byte[], byte[]> producer = producerFactory.createProducer();
|
||||
List<PartitionInfo> partitionsFor = producer
|
||||
.partitionsFor(destination.getName());
|
||||
producer.close();
|
||||
return partitionsFor;
|
||||
}, destination.getName());
|
||||
|
||||
this.topicsInUse.put(destination.getName(),
|
||||
new TopicInformation(null, partitions, false));
|
||||
|
||||
SenderOptions<Object, Object> opts = this.senderOptionsCustomizer.apply(producerProperties.getBindingName(),
|
||||
SenderOptions.create(configs));
|
||||
// TODO bean for converter; MCB doesn't use one on the producer side.
|
||||
RecordMessageConverter converter = new MessagingMessageConverter();
|
||||
AbstractApplicationContext applicationContext = getApplicationContext();
|
||||
FluxMessageChannel resultChannel = null;
|
||||
String channelName = producerProperties.getExtension().getRecordMetadataChannel();
|
||||
if (channelName != null && applicationContext.containsBean(channelName)) {
|
||||
resultChannel = applicationContext.getBean(channelName, FluxMessageChannel.class);
|
||||
}
|
||||
return new ReactorMessageHandler(opts, converter, destination.getName(), resultChannel);
|
||||
}
|
||||
|
||||
// TODO: Refactor to provide in a common area since KafkaMessageChannelBinder also provides this.
|
||||
public void processTopic(final String group, final ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties,
|
||||
final ConsumerFactory<?, ?> consumerFactory, int partitionCount,
|
||||
boolean usingPatterns, boolean groupManagement, String topic) {
|
||||
Collection<PartitionInfo> listenedPartitions;
|
||||
Collection<PartitionInfo> allPartitions = usingPatterns ? Collections.emptyList()
|
||||
: getPartitionInfo(topic, extendedConsumerProperties, consumerFactory,
|
||||
partitionCount);
|
||||
|
||||
if (groupManagement || extendedConsumerProperties.getInstanceCount() == 1) {
|
||||
listenedPartitions = allPartitions;
|
||||
}
|
||||
else {
|
||||
listenedPartitions = new ArrayList<>();
|
||||
for (PartitionInfo partition : allPartitions) {
|
||||
// divide partitions across modules
|
||||
if ((partition.partition() % extendedConsumerProperties
|
||||
.getInstanceCount()) == extendedConsumerProperties
|
||||
.getInstanceIndex()) {
|
||||
listenedPartitions.add(partition);
|
||||
}
|
||||
}
|
||||
}
|
||||
this.topicsInUse.put(topic,
|
||||
new TopicInformation(group, listenedPartitions, usingPatterns));
|
||||
}
|
||||
|
||||
private Collection<PartitionInfo> getPartitionInfo(String topic,
|
||||
final ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties,
|
||||
final ConsumerFactory<?, ?> consumerFactory, int partitionCount) {
|
||||
return provisioningProvider.getPartitionsForTopic(partitionCount,
|
||||
extendedConsumerProperties.getExtension().isAutoRebalanceEnabled(),
|
||||
() -> {
|
||||
try (Consumer<?, ?> consumer = consumerFactory.createConsumer()) {
|
||||
return consumer.partitionsFor(topic);
|
||||
}
|
||||
}, topic);
|
||||
}
|
||||
|
||||
Map<String, TopicInformation> getTopicsInUse() {
|
||||
return this.topicsInUse;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected MessageProducer createConsumerEndpoint(ConsumerDestination destination, String group,
|
||||
ExtendedConsumerProperties<KafkaConsumerProperties> properties) {
|
||||
|
||||
boolean anonymous = !StringUtils.hasText(group);
|
||||
String consumerGroup = anonymous ? "anonymous." + UUID.randomUUID() : group;
|
||||
Map<String, Object> configs = BindingUtils.createConsumerConfigs(anonymous, consumerGroup, properties,
|
||||
this.configurationProperties);
|
||||
|
||||
String destinations = destination.getName();
|
||||
if (this.consumerConfigCustomizer != null) {
|
||||
this.consumerConfigCustomizer.configure(configs, properties.getBindingName(), destinations);
|
||||
}
|
||||
|
||||
MessageConverter converter = BindingUtils.getConsumerMessageConverter(getApplicationContext(), properties,
|
||||
this.configurationProperties);
|
||||
Assert.isInstanceOf(RecordMessageConverter.class, converter);
|
||||
/*
|
||||
* No need to check multiplex here because, if false, the topics are bound one-at-a-time;
|
||||
* it is still required by the provisioner, however.
|
||||
*/
|
||||
List<String> destList = Arrays.stream(StringUtils.commaDelimitedListToStringArray(destinations))
|
||||
.map(String::trim)
|
||||
.toList();
|
||||
ReceiverOptions<Object, Object> opts = ReceiverOptions.create(configs)
|
||||
.addAssignListener(parts -> logger.info("Assigned: " + parts));
|
||||
if (properties.getExtension().isDestinationIsPattern()) {
|
||||
opts = opts.subscription(Pattern.compile(destinations));
|
||||
}
|
||||
else {
|
||||
opts = opts.subscription(destList);
|
||||
}
|
||||
opts = this.receiverOptionsCustomizer.apply(properties.getBindingName(), opts);
|
||||
ReceiverOptions<Object, Object> finalOpts = opts;
|
||||
|
||||
Map<String, Object> props = BindingUtils.createConsumerConfigs(anonymous, consumerGroup, properties,
|
||||
this.configurationProperties);
|
||||
|
||||
DefaultKafkaConsumerFactory<Object, Object> factory = new DefaultKafkaConsumerFactory<>(props);
|
||||
int partitionCount = properties.getInstanceCount() * properties.getConcurrency();
|
||||
boolean groupManagement = properties.getExtension().isAutoRebalanceEnabled();
|
||||
processTopic(consumerGroup, properties, factory, partitionCount, properties.getExtension().isDestinationIsPattern(),
|
||||
groupManagement, destination.getName());
|
||||
|
||||
class ReactorMessageProducer extends MessageProducerSupport {
|
||||
|
||||
private final List<KafkaReceiver<Object, Object>> receivers = new ArrayList<>();
|
||||
|
||||
ReactorMessageProducer() {
|
||||
for (int i = 0; i < properties.getConcurrency(); i++) {
|
||||
this.receivers.add(KafkaReceiver.create(finalOpts));
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
protected void doStart() {
|
||||
List<Flux<Message<Object>>> fluxes = new ArrayList<>();
|
||||
int concurrency = properties.getConcurrency();
|
||||
boolean autoCommit = properties.getExtension().isReactiveAutoCommit();
|
||||
boolean atMostOnce = properties.getExtension().isReactiveAtMostOnce();
|
||||
Assert.state(!(autoCommit && atMostOnce),
|
||||
"Cannot set both reactiveAutoCommit and reactiveAtMostOnce");
|
||||
for (int i = 0; i < concurrency; i++) {
|
||||
Flux<? extends ConsumerRecord<Object, Object>> receive = null;
|
||||
KafkaReceiver<Object, Object> kafkaReceiver = this.receivers.get(i);
|
||||
if (atMostOnce) {
|
||||
receive = kafkaReceiver
|
||||
.receiveAtmostOnce();
|
||||
}
|
||||
else if (!autoCommit) {
|
||||
receive = kafkaReceiver
|
||||
.receive();
|
||||
}
|
||||
if (autoCommit) {
|
||||
fluxes.add(kafkaReceiver
|
||||
.receiveAutoAck()
|
||||
.map(inner -> new GenericMessage<>(inner)));
|
||||
}
|
||||
else {
|
||||
fluxes.add(receive
|
||||
.map(record -> {
|
||||
Message<Object> message = (Message<Object>) ((RecordMessageConverter) converter)
|
||||
.toMessage(record, null, null, null);
|
||||
return addAckHeaderIfNeeded(atMostOnce, record, message);
|
||||
}));
|
||||
}
|
||||
}
|
||||
if (concurrency == 1) {
|
||||
subscribeToPublisher(fluxes.get(0));
|
||||
}
|
||||
else {
|
||||
subscribeToPublisher(Flux.merge(fluxes));
|
||||
}
|
||||
}
|
||||
|
||||
private Message<Object> addAckHeaderIfNeeded(boolean autoCommit, ConsumerRecord<Object, Object> record,
|
||||
Message<Object> message) {
|
||||
|
||||
if (!autoCommit) {
|
||||
if (message.getHeaders() instanceof KafkaMessageHeaders headers) {
|
||||
headers.getRawHeaders().put(KafkaHeaders.ACKNOWLEDGMENT,
|
||||
((ReceiverRecord<Object, Object>) record).receiverOffset());
|
||||
}
|
||||
else {
|
||||
message = MessageBuilder.fromMessage(message)
|
||||
.setHeader(KafkaHeaders.ACKNOWLEDGMENT,
|
||||
((ReceiverRecord<Object, Object>) record).receiverOffset())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
return message;
|
||||
}
|
||||
|
||||
}
|
||||
ReactorMessageProducer reactorMessageProducer = new ReactorMessageProducer();
|
||||
this.messageProducers.put(consumerGroup, reactorMessageProducer);
|
||||
return reactorMessageProducer;
|
||||
}
|
||||
|
||||
public Map<String, MessageProducerSupport> getMessageProducers() {
|
||||
return this.messageProducers;
|
||||
}
|
||||
|
||||
@Override
|
||||
public KafkaConsumerProperties getExtendedConsumerProperties(String channelName) {
|
||||
return this.extendedBindingProperties.getExtendedConsumerProperties(channelName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public KafkaProducerProperties getExtendedProducerProperties(String channelName) {
|
||||
return this.extendedBindingProperties.getExtendedProducerProperties(channelName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDefaultsPrefix() {
|
||||
return this.extendedBindingProperties.getDefaultsPrefix();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<? extends BinderSpecificPropertiesProvider> getExtendedPropertiesEntryClass() {
|
||||
return this.extendedBindingProperties.getExtendedPropertiesEntryClass();
|
||||
}
|
||||
|
||||
public void setExtendedBindingProperties(
|
||||
KafkaExtendedBindingProperties extendedBindingProperties) {
|
||||
|
||||
this.extendedBindingProperties = extendedBindingProperties;
|
||||
}
|
||||
|
||||
private static class ReactorMessageHandler extends AbstractMessageHandler implements Lifecycle {
|
||||
|
||||
private final RecordMessageConverter converter;
|
||||
|
||||
private final String topic;
|
||||
|
||||
private final SenderOptions<Object, Object> senderOptions;
|
||||
|
||||
@Nullable
|
||||
private final FluxMessageChannel results;
|
||||
|
||||
private volatile KafkaSender<Object, Object> sender;
|
||||
|
||||
private volatile boolean running;
|
||||
|
||||
ReactorMessageHandler(SenderOptions<Object, Object> opts, RecordMessageConverter converter,
|
||||
String topic, @Nullable FluxMessageChannel results) {
|
||||
|
||||
this.senderOptions = opts;
|
||||
this.converter = converter;
|
||||
this.topic = topic;
|
||||
this.results = results;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void handleMessageInternal(Message<?> message) {
|
||||
if (this.sender != null) {
|
||||
Object correlation = message.getHeaders().get(IntegrationMessageHeaderAccessor.CORRELATION_ID);
|
||||
if (correlation == null) {
|
||||
correlation = UUID.randomUUID();
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
SenderRecord<Object, Object, Object> sr = SenderRecord.create(
|
||||
(ProducerRecord<Object, Object>) converter.fromMessage(message, topic), correlation);
|
||||
Flux<SenderResult<Object>> result = sender.send(Flux.just(sr));
|
||||
result.subscribe(res -> {
|
||||
if (this.results != null) {
|
||||
this.results.send(MessageBuilder.withPayload(res)
|
||||
.copyHeaders(message.getHeaders())
|
||||
.build());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void start() {
|
||||
if (!this.running) {
|
||||
this.sender = KafkaSender.create(this.senderOptions);
|
||||
this.running = true;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void stop() {
|
||||
if (this.running) {
|
||||
KafkaSender<Object, Object> theSender = this.sender;
|
||||
this.sender = null;
|
||||
theSender.close();
|
||||
this.running = false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isRunning() {
|
||||
return this.running;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.reactorkafka;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.Binder;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaExtendedBindingProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.AdminClientConfigCustomizer;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
|
||||
import org.springframework.cloud.stream.binder.kafka.support.ConsumerConfigCustomizer;
|
||||
import org.springframework.cloud.stream.binder.kafka.support.ProducerConfigCustomizer;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* Binder configuration for ReactorKafka.
|
||||
*
|
||||
* @author Gary Russell
|
||||
* @author Chris Bono
|
||||
*/
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@ConditionalOnMissingBean(Binder.class)
|
||||
@EnableConfigurationProperties({ KafkaProperties.class, KafkaExtendedBindingProperties.class })
|
||||
public class ReactorKafkaBinderConfiguration {
|
||||
|
||||
/**
|
||||
* @ConfigurationProperties is declared on the @Bean method for Spring Boot to ignore
|
||||
* constructor binding on KafkaBinderConfigurationProperties. If constructor binding is
|
||||
* used, it ignores all the JavaBeans style properties when generating configuration metadata.
|
||||
*
|
||||
* See the following issues for more details:
|
||||
*
|
||||
* https://github.com/spring-cloud/spring-cloud-stream/issues/2640
|
||||
* https://github.com/spring-projects/spring-boot/issues/34031
|
||||
*
|
||||
* @param kafkaProperties Spring Kafka properties autoconfigured by Spring Boot
|
||||
*/
|
||||
@Bean
|
||||
@ConfigurationProperties(prefix = "spring.cloud.stream.kafka.binder")
|
||||
KafkaBinderConfigurationProperties configurationProperties(
|
||||
KafkaProperties kafkaProperties) {
|
||||
return new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
}
|
||||
|
||||
@Bean
|
||||
KafkaTopicProvisioner provisioningProvider(
|
||||
KafkaBinderConfigurationProperties configurationProperties,
|
||||
ObjectProvider<AdminClientConfigCustomizer> adminClientConfigCustomizer, KafkaProperties kafkaProperties) {
|
||||
return new KafkaTopicProvisioner(configurationProperties,
|
||||
kafkaProperties, adminClientConfigCustomizer.getIfUnique());
|
||||
}
|
||||
|
||||
@Bean
|
||||
ReactorKafkaBinder reactorKafkaBinder(KafkaBinderConfigurationProperties configurationProperties,
|
||||
KafkaTopicProvisioner provisioningProvider,
|
||||
KafkaExtendedBindingProperties extendedBindingProperties,
|
||||
ObjectProvider<ConsumerConfigCustomizer> consumerConfigCustomizer,
|
||||
ObjectProvider<ProducerConfigCustomizer> producerConfigCustomizer,
|
||||
ObjectProvider<ReceiverOptionsCustomizer> receiverOptionsCustomizers,
|
||||
ObjectProvider<SenderOptionsCustomizer> senderOptionsptionsCustomizers) {
|
||||
|
||||
ReactorKafkaBinder reactorKafkaBinder = new ReactorKafkaBinder(configurationProperties, provisioningProvider);
|
||||
reactorKafkaBinder.setExtendedBindingProperties(extendedBindingProperties);
|
||||
reactorKafkaBinder.setConsumerConfigCustomizer(consumerConfigCustomizer.getIfUnique());
|
||||
reactorKafkaBinder.setProducerConfigCustomizer(producerConfigCustomizer.getIfUnique());
|
||||
reactorKafkaBinder.receiverOptionsCustomizers(receiverOptionsCustomizers);
|
||||
reactorKafkaBinder.senderOptionsCustomizers(senderOptionsptionsCustomizers);
|
||||
return reactorKafkaBinder;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
/*
|
||||
* Copyright 2023-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.reactorkafka;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
import org.springframework.boot.actuate.health.Health;
|
||||
import org.springframework.boot.actuate.health.Status;
|
||||
import org.springframework.cloud.stream.binder.kafka.common.AbstractKafkaBinderHealthIndicator;
|
||||
import org.springframework.cloud.stream.binder.kafka.common.TopicInformation;
|
||||
import org.springframework.integration.endpoint.MessageProducerSupport;
|
||||
import org.springframework.kafka.core.ConsumerFactory;
|
||||
import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
|
||||
|
||||
/**
|
||||
* {@link org.springframework.boot.actuate.health.HealthIndicator} for Reactor Kafka Binder.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
public class ReactorKafkaBinderHealthIndicator extends AbstractKafkaBinderHealthIndicator {
|
||||
|
||||
private final ReactorKafkaBinder binder;
|
||||
|
||||
public ReactorKafkaBinderHealthIndicator(ReactorKafkaBinder binder, ConsumerFactory<?, ?> consumerFactory) {
|
||||
super(consumerFactory);
|
||||
this.binder = binder;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ExecutorService createHealthBinderExecutorService() {
|
||||
return Executors.newSingleThreadExecutor(
|
||||
new CustomizableThreadFactory("reactor-kafka-binder-health-"));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<String, TopicInformation> getTopicsInUse() {
|
||||
return this.binder.getTopicsInUse();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Health buildBinderSpecificHealthDetails() {
|
||||
Map<String, MessageProducerSupport> messageProducerSupportInfo = binder.getMessageProducers();
|
||||
if (messageProducerSupportInfo.isEmpty()) {
|
||||
return Health.unknown().build();
|
||||
}
|
||||
|
||||
Status status = Status.UP;
|
||||
List<Map<String, Object>> messageProducers = new ArrayList<>();
|
||||
|
||||
Map<String, Object> messageProducerDetails = new HashMap<>();
|
||||
for (String groupId : messageProducerSupportInfo.keySet()) {
|
||||
MessageProducerSupport messageProducerSupport = messageProducerSupportInfo.get(groupId);
|
||||
boolean isRunning = messageProducerSupport.isRunning();
|
||||
boolean isOk = messageProducerSupport.isActive();
|
||||
if (!isOk) {
|
||||
status = Status.DOWN;
|
||||
}
|
||||
messageProducerDetails.put("isRunning", isRunning);
|
||||
messageProducerDetails.put("isStoppedAbnormally", !isRunning && !isOk);
|
||||
//messageProducerDetails.put("isPaused", messageProducerSupport.isPaused());
|
||||
messageProducerDetails.put("messageProducerId", messageProducerSupport.getApplicationContextId());
|
||||
messageProducerDetails.put("groupId", groupId);
|
||||
}
|
||||
messageProducers.add(messageProducerDetails);
|
||||
return Health.status(status)
|
||||
.withDetail("messageProducers", messageProducers)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
/*
|
||||
* Copyright 2023-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.reactorkafka;
|
||||
|
||||
import java.util.function.BiFunction;
|
||||
|
||||
import reactor.kafka.receiver.ReceiverOptions;
|
||||
|
||||
import org.springframework.core.Ordered;
|
||||
|
||||
/**
|
||||
* Customizer for {@link ReceiverOptions}; the first parameter contains the binder name;
|
||||
* the second is the {@link ReceiverOptions} to customize. Return the customized
|
||||
* {@link ReceiverOptions}. Applied in {@link Ordered order} if multiple customizers are
|
||||
* found.
|
||||
* <p>
|
||||
* Generic since 4.0.3 to allow customization of deserializers.
|
||||
*
|
||||
* @param <K> the key type.
|
||||
* @param <V> the value type.
|
||||
*
|
||||
* @author Gary Russell
|
||||
* @since 4.0.2
|
||||
*
|
||||
*/
|
||||
public interface ReceiverOptionsCustomizer<K, V>
|
||||
extends BiFunction<String, ReceiverOptions<K, V>, ReceiverOptions<K, V>>, Ordered {
|
||||
|
||||
@Override
|
||||
default int getOrder() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
/*
|
||||
* Copyright 2023-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.reactorkafka;
|
||||
|
||||
import java.util.function.BiFunction;
|
||||
|
||||
import reactor.kafka.sender.SenderOptions;
|
||||
|
||||
import org.springframework.core.Ordered;
|
||||
|
||||
/**
|
||||
* Customizer for {@link SenderOptions}; the first parameter contains the binder name; the
|
||||
* second is the {@link SenderOptions} to customize. Return the customized
|
||||
* {@link SenderOptions}. Applied in {@link Ordered order} if multiple customizers are
|
||||
* found.
|
||||
* <p>
|
||||
* Generic since 4.0.3 to allow customization of serializers.
|
||||
*
|
||||
* @param <K> the key type.
|
||||
* @param <V> the value type.
|
||||
*
|
||||
* @author Gary Russell
|
||||
* @since 4.0.2
|
||||
*
|
||||
*/
|
||||
public interface SenderOptionsCustomizer<K, V>
|
||||
extends BiFunction<String, SenderOptions<K, V>, SenderOptions<K, V>>, Ordered {
|
||||
|
||||
@Override
|
||||
default int getOrder() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Provides classes reactor-kafka binder support.
|
||||
*/
|
||||
package org.springframework.cloud.stream.binder.reactorkafka;
|
||||
@@ -1,2 +0,0 @@
|
||||
reactorKafka:\
|
||||
org.springframework.cloud.stream.binder.reactorkafka.ReactorKafkaBinderConfiguration
|
||||
@@ -1,2 +0,0 @@
|
||||
org.springframework.boot.env.EnvironmentPostProcessor:\
|
||||
org.springframework.cloud.stream.binder.kafka.common.KafkaBinderEnvironmentPostProcessor
|
||||
@@ -1,100 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.reactorkafka;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.kafka.clients.consumer.KafkaConsumer;
|
||||
import org.apache.kafka.common.Node;
|
||||
import org.apache.kafka.common.PartitionInfo;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInstance;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
|
||||
import org.springframework.boot.actuate.health.Health;
|
||||
import org.springframework.boot.actuate.health.Status;
|
||||
import org.springframework.cloud.stream.binder.kafka.common.TopicInformation;
|
||||
import org.springframework.integration.endpoint.MessageProducerSupport;
|
||||
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
|
||||
|
||||
import static java.util.Collections.singleton;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
public class ReactorKafkaBinderHealthIndicatorTests {
|
||||
|
||||
private static final String TEST_TOPIC = "test";
|
||||
|
||||
private ReactorKafkaBinderHealthIndicator indicator;
|
||||
|
||||
@Mock
|
||||
private DefaultKafkaConsumerFactory<?, ?> consumerFactory;
|
||||
|
||||
@Mock
|
||||
private KafkaConsumer consumer;
|
||||
|
||||
@Mock
|
||||
MessageProducerSupport messageProducerSupport1;
|
||||
|
||||
@Mock
|
||||
private ReactorKafkaBinder binder;
|
||||
|
||||
private final Map<String, TopicInformation> topicsInUse = new HashMap<>();
|
||||
|
||||
@BeforeEach
|
||||
public void setup() {
|
||||
MockitoAnnotations.openMocks(this);
|
||||
org.mockito.BDDMockito.given(consumerFactory.createConsumer())
|
||||
.willReturn((consumer));
|
||||
org.mockito.BDDMockito.given(binder.getTopicsInUse()).willReturn(topicsInUse);
|
||||
this.indicator = new ReactorKafkaBinderHealthIndicator(binder, consumerFactory);
|
||||
this.indicator.setTimeout(10);
|
||||
}
|
||||
|
||||
@Test
|
||||
void reactorKafkaBinderIsUp() {
|
||||
final List<PartitionInfo> partitions = partitions(new Node(0, null, 0));
|
||||
topicsInUse.put(TEST_TOPIC, new TopicInformation(
|
||||
"group1-healthIndicator", partitions, false));
|
||||
org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC))
|
||||
.willReturn(partitions);
|
||||
org.mockito.BDDMockito.given(binder.getMessageProducers())
|
||||
.willReturn(Map.of("group1-healthIndicator", messageProducerSupport1));
|
||||
org.mockito.BDDMockito.given(messageProducerSupport1.isRunning()).willReturn(true);
|
||||
org.mockito.BDDMockito.given(messageProducerSupport1.isActive()).willReturn(true);
|
||||
Health health = indicator.health();
|
||||
assertThat(health.getStatus()).isEqualTo(Status.UP);
|
||||
assertThat(health.getDetails()).containsEntry("topicsInUse", singleton(TEST_TOPIC));
|
||||
assertThat(health.getDetails()).hasEntrySatisfying("messageProducers", value ->
|
||||
assertThat((ArrayList<?>) value).hasSize(1));
|
||||
}
|
||||
|
||||
private List<PartitionInfo> partitions(Node leader) {
|
||||
List<PartitionInfo> partitions = new ArrayList<>();
|
||||
partitions.add(new PartitionInfo(TEST_TOPIC, 0, leader, null, null));
|
||||
return partitions;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,242 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.reactorkafka;
|
||||
|
||||
import java.lang.reflect.Type;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.apache.kafka.clients.consumer.Consumer;
|
||||
import org.apache.kafka.clients.consumer.ConsumerConfig;
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
import org.apache.kafka.clients.producer.ProducerRecord;
|
||||
import org.apache.kafka.common.serialization.StringDeserializer;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.kafka.receiver.ReceiverOptions;
|
||||
import reactor.kafka.receiver.ReceiverRecord;
|
||||
import reactor.kafka.sender.SenderResult;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.WebApplicationType;
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.cloud.stream.function.StreamBridge;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.integration.annotation.ServiceActivator;
|
||||
import org.springframework.integration.channel.FluxMessageChannel;
|
||||
import org.springframework.integration.support.MessageBuilder;
|
||||
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
|
||||
import org.springframework.kafka.support.Acknowledgment;
|
||||
import org.springframework.kafka.support.converter.MessagingMessageConverter;
|
||||
import org.springframework.kafka.support.converter.RecordMessageConverter;
|
||||
import org.springframework.kafka.test.EmbeddedKafkaBroker;
|
||||
import org.springframework.kafka.test.context.EmbeddedKafka;
|
||||
import org.springframework.kafka.test.utils.KafkaTestUtils;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.awaitility.Awaitility.await;
|
||||
|
||||
/**
|
||||
* Integration tests for {@link ReactorKafkaBinder}.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @author Gary Russell
|
||||
* @author Chris Bono
|
||||
*/
|
||||
@ExtendWith(SpringExtension.class)
|
||||
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD)
|
||||
@EmbeddedKafka(topics = { "uppercased-words", "lowercased-words" })
|
||||
class ReactorKafkaBinderIntegrationTests {
|
||||
|
||||
private static final List<String> recOptsCustOrder = Collections.synchronizedList(new ArrayList<>());
|
||||
|
||||
private static final List<String> patternedDeliveries = Collections.synchronizedList(new ArrayList<>());
|
||||
|
||||
@Autowired
|
||||
private EmbeddedKafkaBroker embeddedKafka;
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(booleans = { false, true })
|
||||
void endToEndReactorKafkaBinder(boolean excludeKafkaAutoConfig) {
|
||||
|
||||
recOptsCustOrder.clear();
|
||||
patternedDeliveries.clear();
|
||||
Map<String, Object> consumerProps = KafkaTestUtils.consumerProps("group1", "false", embeddedKafka);
|
||||
consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
|
||||
DefaultKafkaConsumerFactory<String, String> cf = new DefaultKafkaConsumerFactory<>(consumerProps);
|
||||
Consumer<String, String> consumer1 = cf.createConsumer();
|
||||
embeddedKafka.consumeFromEmbeddedTopics(consumer1, "uppercased-words");
|
||||
Consumer<String, String> consumer2 = cf.createConsumer("group2", null);
|
||||
embeddedKafka.consumeFromEmbeddedTopics(consumer2, "lowercased-words");
|
||||
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(ReactiveKafkaApplication.class)
|
||||
.web(WebApplicationType.NONE).run(
|
||||
"--server.port=0",
|
||||
"--spring.jmx.enabled=false",
|
||||
"--spring.kafka.consumer.metadata.max.age.ms=1000",
|
||||
"--spring.cloud.function.definition=uppercase;lowercase;patternConsumer",
|
||||
"--spring.cloud.stream.function.reactive.uppercase=true",
|
||||
"--spring.cloud.stream.function.reactive.lowercase=true",
|
||||
"--spring.cloud.stream.bindings.uppercase-in-0.group=grp1",
|
||||
"--spring.cloud.stream.bindings.uppercase-in-0.destination=words1",
|
||||
"--spring.cloud.stream.bindings.uppercase-out-0.destination=uppercased-words",
|
||||
"--spring.cloud.stream.bindings.lowercase-in-0.group=grp2",
|
||||
"--spring.cloud.stream.bindings.lowercase-in-0.destination=words2",
|
||||
"--spring.cloud.stream.bindings.lowercase-out-0.destination=lowercased-words",
|
||||
"--spring.cloud.stream.bindings.patternConsumer-in-0.group=grp3",
|
||||
"--spring.cloud.stream.bindings.patternConsumer-in-0.destination=.*-words",
|
||||
"--spring.cloud.stream.kafka.bindings.words1.producer.record-metadata-channel=sendResults",
|
||||
"--spring.cloud.stream.kafka.bindings.words2.producer.record-metadata-channel=sendResults",
|
||||
"--spring.cloud.stream.kafka.bindings.patternConsumer-in-0.consumer.destination-is-pattern=true",
|
||||
"--spring.cloud.stream.kafka.bindings.lowercase-in-0.consumer.converterBeanName=fullRR",
|
||||
"--spring.cloud.stream.kafka.binder.brokers=" + embeddedKafka.getBrokersAsString(),
|
||||
excludeKafkaAutoConfigParam(excludeKafkaAutoConfig))) {
|
||||
|
||||
StreamBridge streamBridge = context.getBean(StreamBridge.class);
|
||||
streamBridge.send("words1", MessageBuilder.withPayload("foobar")
|
||||
.setCorrelationId(42)
|
||||
.build());
|
||||
streamBridge.send("words2", MessageBuilder.withPayload("BAZQUX")
|
||||
.setCorrelationId(43)
|
||||
.build());
|
||||
|
||||
assertThat(KafkaTestUtils.getSingleRecord(consumer1, "uppercased-words"))
|
||||
.isNotNull()
|
||||
.extracting(ConsumerRecord::value)
|
||||
.isEqualTo("FOOBAR");
|
||||
|
||||
assertThat(KafkaTestUtils.getSingleRecord(consumer2, "lowercased-words"))
|
||||
.isNotNull()
|
||||
.extracting(ConsumerRecord::value)
|
||||
.isEqualTo("bazqux");
|
||||
|
||||
assertThat(recOptsCustOrder).containsExactly("two", "one", "two", "one", "two", "one");
|
||||
await().untilAsserted(() -> assertThat(patternedDeliveries).contains("bazqux", "FOOBAR"));
|
||||
assertThat(context.getBean(ReactiveKafkaApplication.class).correlation).contains(42, 43);
|
||||
}
|
||||
}
|
||||
|
||||
private String excludeKafkaAutoConfigParam(boolean excludeKafkaAutoConfig) {
|
||||
return excludeKafkaAutoConfig ?
|
||||
"--spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration" : "foo=bar";
|
||||
}
|
||||
|
||||
@EnableAutoConfiguration
|
||||
public static class ReactiveKafkaApplication {
|
||||
|
||||
final List<Integer> correlation = Collections.synchronizedList(new ArrayList<>());
|
||||
|
||||
@Bean
|
||||
RecordMessageConverter fullRR() {
|
||||
return new RecordMessageConverter() {
|
||||
|
||||
private final RecordMessageConverter converter = new MessagingMessageConverter();
|
||||
|
||||
@Override
|
||||
public Message<?> toMessage(ConsumerRecord<?, ?> record, Acknowledgment acknowledgment,
|
||||
Consumer<?, ?> consumer, Type payloadType) {
|
||||
|
||||
return MessageBuilder.withPayload(record).build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProducerRecord<?, ?> fromMessage(Message<?> message, String defaultTopic) {
|
||||
return this.converter.fromMessage(message, defaultTopic);
|
||||
}
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
@Bean
|
||||
Function<Flux<String>, Flux<String>> uppercase() {
|
||||
return s -> s.map(String::toUpperCase);
|
||||
}
|
||||
|
||||
@Bean
|
||||
Function<Flux<ReceiverRecord<byte[], byte[]>>, Flux<String>> lowercase() {
|
||||
return s -> s.map(rec -> new String(rec.value()).toLowerCase());
|
||||
}
|
||||
|
||||
@Bean
|
||||
java.util.function.Consumer<Flux<String>> patternConsumer() {
|
||||
return f -> f.doOnNext(s -> patternedDeliveries.add(s))
|
||||
.subscribe();
|
||||
}
|
||||
|
||||
@Bean
|
||||
ReceiverOptionsCustomizer<?, ?> cust1() {
|
||||
return (t, u) -> {
|
||||
recOptsCustOrder.add("one");
|
||||
return u;
|
||||
};
|
||||
}
|
||||
|
||||
@Bean
|
||||
ReceiverOptionsCustomizer<String, byte[]> cust2() {
|
||||
return new ReceiverOptionsCustomizer<>() {
|
||||
|
||||
@Override
|
||||
public ReceiverOptions<String, byte[]> apply(String t, ReceiverOptions<String, byte[]> u) {
|
||||
recOptsCustOrder.add("two");
|
||||
u.withKeyDeserializer(new StringDeserializer());
|
||||
return u;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getOrder() {
|
||||
return -1;
|
||||
}
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
@Bean
|
||||
FluxMessageChannel sendResults() {
|
||||
return new FluxMessageChannel();
|
||||
}
|
||||
|
||||
@ServiceActivator(inputChannel = "sendResults")
|
||||
void handleResults(SenderResult<Integer> result) {
|
||||
if (result.exception() != null) {
|
||||
failureFor(result);
|
||||
}
|
||||
else {
|
||||
successFor(result);
|
||||
}
|
||||
}
|
||||
|
||||
private void failureFor(SenderResult<Integer> result) {
|
||||
this.correlation.clear();
|
||||
}
|
||||
|
||||
private boolean successFor(SenderResult<Integer> result) {
|
||||
return this.correlation.add(result.correlationMetadata());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,352 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.reactorkafka;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
import org.apache.kafka.clients.producer.RecordMetadata;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.reactivestreams.Subscriber;
|
||||
import org.reactivestreams.Subscription;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.kafka.receiver.ReceiverOffset;
|
||||
import reactor.kafka.sender.SenderResult;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
|
||||
import org.springframework.cloud.stream.binder.Binding;
|
||||
import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
|
||||
import org.springframework.context.support.GenericApplicationContext;
|
||||
import org.springframework.integration.IntegrationMessageHeaderAccessor;
|
||||
import org.springframework.integration.channel.FluxMessageChannel;
|
||||
import org.springframework.integration.config.ConsumerEndpointFactoryBean;
|
||||
import org.springframework.integration.support.MessageBuilder;
|
||||
import org.springframework.kafka.core.DefaultKafkaProducerFactory;
|
||||
import org.springframework.kafka.core.KafkaTemplate;
|
||||
import org.springframework.kafka.support.KafkaHeaders;
|
||||
import org.springframework.kafka.test.condition.EmbeddedKafkaCondition;
|
||||
import org.springframework.kafka.test.context.EmbeddedKafka;
|
||||
import org.springframework.kafka.test.utils.KafkaTestUtils;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.MessageChannel;
|
||||
import org.springframework.messaging.MessageHandler;
|
||||
import org.springframework.messaging.MessagingException;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.BDDMockito.given;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
/**
|
||||
* @author Gary Russell
|
||||
* @since 4.0
|
||||
*
|
||||
*/
|
||||
@EmbeddedKafka(topics = { "testCa", "testCb", "testC1", "testP" })
|
||||
public class ReactorKafkaBinderTests {
|
||||
|
||||
@SuppressWarnings({ "rawtypes", "unchecked" })
|
||||
@Test
|
||||
void consumerBinding() throws Exception {
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
kafkaProperties.setBootstrapServers(
|
||||
Collections.singletonList(EmbeddedKafkaCondition.getBroker().getBrokersAsString()));
|
||||
KafkaBinderConfigurationProperties binderProps = new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
KafkaTopicProvisioner provisioner = new KafkaTopicProvisioner(binderProps, kafkaProperties, prop -> {
|
||||
});
|
||||
provisioner.setMetadataRetryOperations(new RetryTemplate());
|
||||
ReactorKafkaBinder binder = new ReactorKafkaBinder(binderProps, provisioner);
|
||||
binder.setApplicationContext(mock(GenericApplicationContext.class));
|
||||
|
||||
CountDownLatch latch = new CountDownLatch(2);
|
||||
|
||||
FluxMessageChannel inbound = new FluxMessageChannel();
|
||||
Subscriber<Message<?>> sub = new Subscriber<Message<?>>() {
|
||||
|
||||
@Override
|
||||
public void onSubscribe(Subscription s) {
|
||||
s.request(2);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onNext(Message<?> t) {
|
||||
latch.countDown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(Throwable t) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onComplete() {
|
||||
}
|
||||
|
||||
};
|
||||
inbound.subscribe(sub);
|
||||
|
||||
KafkaConsumerProperties ext = new KafkaConsumerProperties();
|
||||
ExtendedConsumerProperties<KafkaConsumerProperties> props =
|
||||
new ExtendedConsumerProperties<KafkaConsumerProperties>(ext);
|
||||
props.setMultiplex(true);
|
||||
|
||||
Binding<MessageChannel> consumer = binder.bindConsumer("testCa, testCb", "foo", inbound, props);
|
||||
|
||||
DefaultKafkaProducerFactory pf =
|
||||
new DefaultKafkaProducerFactory<>(KafkaTestUtils.producerProps(EmbeddedKafkaCondition.getBroker()));
|
||||
KafkaTemplate kt = new KafkaTemplate<>(pf);
|
||||
kt.send("testCa", "foo").get(10, TimeUnit.SECONDS);
|
||||
kt.send("testCb", "bar").get(10, TimeUnit.SECONDS);
|
||||
assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
consumer.unbind();
|
||||
pf.destroy();
|
||||
}
|
||||
|
||||
@Test
|
||||
void concurrencyManual() throws Exception {
|
||||
concurrency(false);
|
||||
}
|
||||
|
||||
@Test
|
||||
void concurrencyAtMostOnce() throws Exception {
|
||||
concurrency(true);
|
||||
}
|
||||
|
||||
@SuppressWarnings({ "rawtypes", "unchecked" })
|
||||
void concurrency(boolean atMostOnce) throws Exception {
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
kafkaProperties.setBootstrapServers(
|
||||
Collections.singletonList(EmbeddedKafkaCondition.getBroker().getBrokersAsString()));
|
||||
KafkaBinderConfigurationProperties binderProps = new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
KafkaTopicProvisioner provisioner = new KafkaTopicProvisioner(binderProps, kafkaProperties, prop -> {
|
||||
});
|
||||
provisioner.setMetadataRetryOperations(new RetryTemplate());
|
||||
ReactorKafkaBinder binder = new ReactorKafkaBinder(binderProps, provisioner);
|
||||
binder.setApplicationContext(mock(GenericApplicationContext.class));
|
||||
|
||||
CountDownLatch subscriptionLatch = new CountDownLatch(1);
|
||||
CountDownLatch messageLatch1 = new CountDownLatch(4);
|
||||
CountDownLatch messageLatch2 = new CountDownLatch(6);
|
||||
Set<Integer> partitions = new HashSet<>();
|
||||
List<String> payloads = Collections.synchronizedList(new ArrayList<>());
|
||||
|
||||
FluxMessageChannel inbound = new FluxMessageChannel();
|
||||
Subscriber<Message<?>> sub = new Subscriber<Message<?>>() {
|
||||
|
||||
@Override
|
||||
public void onSubscribe(Subscription s) {
|
||||
s.request(10);
|
||||
subscriptionLatch.countDown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onNext(Message<?> msg) {
|
||||
payloads.add((String) msg.getPayload());
|
||||
partitions.add(msg.getHeaders().get(KafkaHeaders.RECEIVED_PARTITION, Integer.class));
|
||||
if (!atMostOnce) {
|
||||
msg.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT, ReceiverOffset.class).acknowledge();
|
||||
}
|
||||
messageLatch1.countDown();
|
||||
messageLatch2.countDown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(Throwable t) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onComplete() {
|
||||
}
|
||||
|
||||
};
|
||||
inbound.subscribe(sub);
|
||||
|
||||
KafkaConsumerProperties ext = new KafkaConsumerProperties();
|
||||
ext.setReactiveAtMostOnce(atMostOnce);
|
||||
ExtendedConsumerProperties<KafkaConsumerProperties> props =
|
||||
new ExtendedConsumerProperties<KafkaConsumerProperties>(ext);
|
||||
props.setConcurrency(2);
|
||||
|
||||
Binding<MessageChannel> consumer = binder.bindConsumer("testC1", "foo", inbound, props);
|
||||
|
||||
assertThat(subscriptionLatch.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
DefaultKafkaProducerFactory pf =
|
||||
new DefaultKafkaProducerFactory<>(KafkaTestUtils.producerProps(EmbeddedKafkaCondition.getBroker()));
|
||||
KafkaTemplate kt = new KafkaTemplate<>(pf);
|
||||
kt.send("testC1", 0, null, "foo").get(10, TimeUnit.SECONDS);
|
||||
kt.send("testC1", 1, null, "bar").get(10, TimeUnit.SECONDS);
|
||||
kt.send("testC1", 0, null, "baz").get(10, TimeUnit.SECONDS);
|
||||
kt.send("testC1", 1, null, "qux").get(10, TimeUnit.SECONDS);
|
||||
assertThat(messageLatch1.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
consumer.stop();
|
||||
consumer.start();
|
||||
kt.send("testC1", 0, null, "fiz").get(10, TimeUnit.SECONDS);
|
||||
kt.send("testC1", 1, null, "buz").get(10, TimeUnit.SECONDS);
|
||||
assertThat(messageLatch2.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
assertThat(partitions).hasSize(2);
|
||||
consumer.unbind();
|
||||
pf.destroy();
|
||||
Collections.sort(payloads);
|
||||
if (!atMostOnce) {
|
||||
assertThat(payloads).containsExactly("bar", "baz", "buz", "fiz", "foo", "qux");
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings({ "rawtypes", "unchecked" })
|
||||
@Test
|
||||
void autoCommit() throws Exception {
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
kafkaProperties.setBootstrapServers(
|
||||
Collections.singletonList(EmbeddedKafkaCondition.getBroker().getBrokersAsString()));
|
||||
KafkaBinderConfigurationProperties binderProps = new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
KafkaTopicProvisioner provisioner = new KafkaTopicProvisioner(binderProps, kafkaProperties, prop -> {
|
||||
});
|
||||
provisioner.setMetadataRetryOperations(new RetryTemplate());
|
||||
ReactorKafkaBinder binder = new ReactorKafkaBinder(binderProps, provisioner);
|
||||
binder.setApplicationContext(mock(GenericApplicationContext.class));
|
||||
|
||||
CountDownLatch subscriptionLatch = new CountDownLatch(1);
|
||||
CountDownLatch messageLatch1 = new CountDownLatch(4);
|
||||
Set<Integer> partitions = new HashSet<>();
|
||||
List<String> payloads = Collections.synchronizedList(new ArrayList<>());
|
||||
|
||||
FluxMessageChannel inbound = new FluxMessageChannel();
|
||||
Subscriber<Message<?>> sub = new Subscriber<Message<?>>() {
|
||||
|
||||
@Override
|
||||
public void onSubscribe(Subscription s) {
|
||||
s.request(10);
|
||||
subscriptionLatch.countDown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onNext(Message<?> msg) {
|
||||
((Message<Flux<ConsumerRecord<?, String>>>) msg).getPayload()
|
||||
.doOnNext(rec -> {
|
||||
payloads.add(rec.value());
|
||||
messageLatch1.countDown();
|
||||
})
|
||||
.subscribe();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(Throwable t) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onComplete() {
|
||||
}
|
||||
|
||||
};
|
||||
inbound.subscribe(sub);
|
||||
|
||||
KafkaConsumerProperties ext = new KafkaConsumerProperties();
|
||||
ext.setReactiveAutoCommit(true);
|
||||
ExtendedConsumerProperties<KafkaConsumerProperties> props =
|
||||
new ExtendedConsumerProperties<KafkaConsumerProperties>(ext);
|
||||
props.setConcurrency(2);
|
||||
|
||||
Binding<MessageChannel> consumer = binder.bindConsumer("testC1", "foo", inbound, props);
|
||||
|
||||
assertThat(subscriptionLatch.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
DefaultKafkaProducerFactory pf =
|
||||
new DefaultKafkaProducerFactory<>(KafkaTestUtils.producerProps(EmbeddedKafkaCondition.getBroker()));
|
||||
KafkaTemplate kt = new KafkaTemplate<>(pf);
|
||||
kt.send("testC1", 0, null, "foo").get(10, TimeUnit.SECONDS);
|
||||
kt.send("testC1", 1, null, "bar").get(10, TimeUnit.SECONDS);
|
||||
kt.send("testC1", 0, null, "baz").get(10, TimeUnit.SECONDS);
|
||||
kt.send("testC1", 1, null, "qux").get(10, TimeUnit.SECONDS);
|
||||
assertThat(messageLatch1.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
consumer.unbind();
|
||||
pf.destroy();
|
||||
Collections.sort(payloads);
|
||||
assertThat(payloads).containsExactly("bar", "baz", "foo", "qux");
|
||||
}
|
||||
|
||||
@Test
|
||||
void producerBinding() throws InterruptedException {
|
||||
KafkaProperties kafkaProperties = new KafkaProperties();
|
||||
kafkaProperties.setBootstrapServers(
|
||||
Collections.singletonList(EmbeddedKafkaCondition.getBroker().getBrokersAsString()));
|
||||
KafkaBinderConfigurationProperties binderProps = new KafkaBinderConfigurationProperties(kafkaProperties);
|
||||
KafkaTopicProvisioner provisioner = new KafkaTopicProvisioner(binderProps, kafkaProperties, prop -> {
|
||||
});
|
||||
provisioner.setMetadataRetryOperations(new RetryTemplate());
|
||||
ReactorKafkaBinder binder = new ReactorKafkaBinder(binderProps, provisioner);
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
GenericApplicationContext context = new GenericApplicationContext();
|
||||
context.registerBean("sendResults", FluxMessageChannel.class);
|
||||
context.refresh();
|
||||
FluxMessageChannel results = context.getBean("sendResults", FluxMessageChannel.class);
|
||||
ConsumerEndpointFactoryBean fb = new ConsumerEndpointFactoryBean();
|
||||
AtomicReference<SenderResult<Integer>> senderResult = new AtomicReference<>();
|
||||
fb.setHandler(new MessageHandler() {
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public void handleMessage(Message<?> message) throws MessagingException {
|
||||
senderResult.set((SenderResult<Integer>) message.getPayload());
|
||||
latch.countDown();
|
||||
}
|
||||
|
||||
});
|
||||
fb.setInputChannel(results);
|
||||
fb.setBeanFactory(context.getBeanFactory());
|
||||
fb.afterPropertiesSet();
|
||||
fb.start();
|
||||
binder.setApplicationContext(context);
|
||||
@SuppressWarnings("rawtypes")
|
||||
ObjectProvider<SenderOptionsCustomizer> cust = mock(ObjectProvider.class);
|
||||
AtomicBoolean custCalled = new AtomicBoolean();
|
||||
given(cust.getIfUnique()).willReturn((name, opts) -> {
|
||||
custCalled.set(true);
|
||||
return opts;
|
||||
});
|
||||
binder.senderOptionsCustomizers(cust);
|
||||
|
||||
MessageChannel outbound = new FluxMessageChannel();
|
||||
KafkaProducerProperties ext = new KafkaProducerProperties();
|
||||
ExtendedProducerProperties<KafkaProducerProperties> props =
|
||||
new ExtendedProducerProperties<KafkaProducerProperties>(ext);
|
||||
props.getExtension().setRecordMetadataChannel("sendResults");
|
||||
|
||||
Binding<MessageChannel> bindProducer = binder.bindProducer("testP", outbound, props);
|
||||
AtomicReference<Mono<RecordMetadata>> sendResult = new AtomicReference<>();
|
||||
outbound.send(MessageBuilder.withPayload("foo")
|
||||
.setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID, 1)
|
||||
.build());
|
||||
assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
assertThat(senderResult.get().correlationMetadata()).isEqualTo(1);
|
||||
bindProducer.unbind();
|
||||
assertThat(custCalled).isTrue();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
<configuration>
|
||||
<appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%d{ISO8601} %5p %t %c{2}:%L - %m%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
<logger name="org.apache.kafka" level="WARN"/>
|
||||
<logger name="reactor.kafka" level="DEBUG"/>
|
||||
<logger name="org.springframework.integration.kafka" level="INFO"/>
|
||||
<logger name="org.springframework.kafka" level="DEBUG"/>
|
||||
<logger name="org.springframework.cloud.stream" level="INFO" />
|
||||
<logger name="org.springframework.integration.channel" level="DEBUG" />
|
||||
<logger name="kafka.server.ReplicaFetcherThread" level="ERROR"/>
|
||||
<logger name="kafka.server.LogDirFailureChannel" level="FATAL"/>
|
||||
<logger name="kafka.server.BrokerMetadataCheckpoint" level="ERROR"/>
|
||||
<logger name="kafka.utils.CoreUtils$" level="ERROR"/>
|
||||
<root level="WARN">
|
||||
<appender-ref ref="stdout"/>
|
||||
</root>
|
||||
</configuration>
|
||||
@@ -1,86 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>spring-cloud-stream-binder-kafka-streams</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>spring-cloud-stream-binder-kafka-streams</name>
|
||||
<description>Kafka Streams Binder Implementation</description>
|
||||
|
||||
<parent>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-binder-kafka-parent</artifactId>
|
||||
<version>4.1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<properties>
|
||||
<avro.version>1.8.2</avro.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-binder-kafka-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-actuator</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-configuration-processor</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-autoconfigure</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-streams</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.kafka</groupId>
|
||||
<artifactId>spring-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.kafka</groupId>
|
||||
<artifactId>spring-kafka-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-autoconfigure-processor</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.awaitility</groupId>
|
||||
<artifactId>awaitility</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.integration</groupId>
|
||||
<artifactId>spring-integration-test</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.apache.avro</groupId>
|
||||
<artifactId>avro-compiler</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-binder-kafka</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -1,663 +0,0 @@
|
||||
/*
|
||||
* Copyright 2019-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.kafka.clients.consumer.ConsumerConfig;
|
||||
import org.apache.kafka.common.header.Header;
|
||||
import org.apache.kafka.common.header.Headers;
|
||||
import org.apache.kafka.common.serialization.Serde;
|
||||
import org.apache.kafka.common.serialization.Serdes;
|
||||
import org.apache.kafka.common.utils.Bytes;
|
||||
import org.apache.kafka.streams.StreamsBuilder;
|
||||
import org.apache.kafka.streams.StreamsConfig;
|
||||
import org.apache.kafka.streams.Topology;
|
||||
import org.apache.kafka.streams.errors.LogAndContinueExceptionHandler;
|
||||
import org.apache.kafka.streams.errors.LogAndFailExceptionHandler;
|
||||
import org.apache.kafka.streams.kstream.Consumed;
|
||||
import org.apache.kafka.streams.kstream.GlobalKTable;
|
||||
import org.apache.kafka.streams.kstream.KStream;
|
||||
import org.apache.kafka.streams.kstream.KTable;
|
||||
import org.apache.kafka.streams.kstream.Materialized;
|
||||
import org.apache.kafka.streams.processor.TimestampExtractor;
|
||||
import org.apache.kafka.streams.processor.api.Processor;
|
||||
import org.apache.kafka.streams.processor.api.ProcessorContext;
|
||||
import org.apache.kafka.streams.processor.api.ProcessorSupplier;
|
||||
import org.apache.kafka.streams.processor.api.Record;
|
||||
import org.apache.kafka.streams.processor.api.RecordMetadata;
|
||||
import org.apache.kafka.streams.state.KeyValueStore;
|
||||
import org.apache.kafka.streams.state.StoreBuilder;
|
||||
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.beans.factory.config.BeanDefinition;
|
||||
import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
|
||||
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
|
||||
import org.springframework.beans.factory.support.BeanDefinitionRegistry;
|
||||
import org.springframework.boot.context.properties.bind.BindContext;
|
||||
import org.springframework.boot.context.properties.bind.BindHandler;
|
||||
import org.springframework.boot.context.properties.bind.Bindable;
|
||||
import org.springframework.boot.context.properties.bind.PropertySourcesPlaceholdersResolver;
|
||||
import org.springframework.boot.context.properties.source.ConfigurationPropertyName;
|
||||
import org.springframework.boot.context.properties.source.ConfigurationPropertySources;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties;
|
||||
import org.springframework.cloud.stream.config.BindingProperties;
|
||||
import org.springframework.cloud.stream.config.BindingServiceProperties;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ApplicationContextAware;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.core.ResolvableType;
|
||||
import org.springframework.core.env.ConfigurableEnvironment;
|
||||
import org.springframework.core.env.MutablePropertySources;
|
||||
import org.springframework.integration.support.utils.IntegrationUtils;
|
||||
import org.springframework.kafka.config.KafkaStreamsConfiguration;
|
||||
import org.springframework.kafka.config.StreamsBuilderFactoryBean;
|
||||
import org.springframework.kafka.config.StreamsBuilderFactoryBeanConfigurer;
|
||||
import org.springframework.kafka.core.CleanupConfig;
|
||||
import org.springframework.kafka.streams.RecoveringDeserializationExceptionHandler;
|
||||
import org.springframework.messaging.MessageHeaders;
|
||||
import org.springframework.messaging.support.MessageBuilder;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.ObjectUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* @author Soby Chacko
|
||||
* @since 3.0.0
|
||||
*/
|
||||
public abstract class AbstractKafkaStreamsBinderProcessor implements ApplicationContextAware {
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(AbstractKafkaStreamsBinderProcessor.class);
|
||||
|
||||
private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue;
|
||||
|
||||
private final BindingServiceProperties bindingServiceProperties;
|
||||
|
||||
private final KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties;
|
||||
|
||||
private final CleanupConfig cleanupConfig;
|
||||
|
||||
private final KeyValueSerdeResolver keyValueSerdeResolver;
|
||||
|
||||
protected ConfigurableApplicationContext applicationContext;
|
||||
|
||||
public AbstractKafkaStreamsBinderProcessor(BindingServiceProperties bindingServiceProperties,
|
||||
KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue,
|
||||
KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties,
|
||||
KeyValueSerdeResolver keyValueSerdeResolver, CleanupConfig cleanupConfig) {
|
||||
this.bindingServiceProperties = bindingServiceProperties;
|
||||
this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue;
|
||||
this.kafkaStreamsExtendedBindingProperties = kafkaStreamsExtendedBindingProperties;
|
||||
this.keyValueSerdeResolver = keyValueSerdeResolver;
|
||||
this.cleanupConfig = cleanupConfig;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void setApplicationContext(ApplicationContext applicationContext)
|
||||
throws BeansException {
|
||||
this.applicationContext = (ConfigurableApplicationContext) applicationContext;
|
||||
}
|
||||
|
||||
protected Topology.AutoOffsetReset getAutoOffsetReset(String inboundName, KafkaStreamsConsumerProperties extendedConsumerProperties) {
|
||||
final KafkaConsumerProperties.StartOffset startOffset = extendedConsumerProperties
|
||||
.getStartOffset();
|
||||
Topology.AutoOffsetReset autoOffsetReset = null;
|
||||
if (startOffset != null) {
|
||||
autoOffsetReset = switch (startOffset) {
|
||||
case earliest -> Topology.AutoOffsetReset.EARLIEST;
|
||||
case latest -> Topology.AutoOffsetReset.LATEST;
|
||||
};
|
||||
}
|
||||
if (extendedConsumerProperties.isResetOffsets()) {
|
||||
AbstractKafkaStreamsBinderProcessor.LOG.warn("Detected resetOffsets configured on binding "
|
||||
+ inboundName + ". "
|
||||
+ "Setting resetOffsets in Kafka Streams binder does not have any effect.");
|
||||
}
|
||||
return autoOffsetReset;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
protected void handleKTableGlobalKTableInputs(Object[] arguments, int index, String input, Class<?> parameterType, Object targetBean,
|
||||
StreamsBuilderFactoryBean streamsBuilderFactoryBean, StreamsBuilder streamsBuilder,
|
||||
KafkaStreamsConsumerProperties extendedConsumerProperties,
|
||||
Serde<?> keySerde, Serde<?> valueSerde, Topology.AutoOffsetReset autoOffsetReset, boolean firstBuild) {
|
||||
if (firstBuild) {
|
||||
addStateStoreBeans(streamsBuilder);
|
||||
}
|
||||
if (parameterType.isAssignableFrom(KTable.class)) {
|
||||
String materializedAs = extendedConsumerProperties.getMaterializedAs();
|
||||
String bindingDestination = this.bindingServiceProperties.getBindingDestination(input);
|
||||
KTable<?, ?> table = getKTable(extendedConsumerProperties, streamsBuilder, keySerde, valueSerde, materializedAs,
|
||||
bindingDestination, autoOffsetReset);
|
||||
KTableBoundElementFactory.KTableWrapper kTableWrapper =
|
||||
(KTableBoundElementFactory.KTableWrapper) targetBean;
|
||||
//wrap the proxy created during the initial target type binding with real object (KTable)
|
||||
kTableWrapper.wrap((KTable<Object, Object>) table);
|
||||
this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding(input, streamsBuilderFactoryBean);
|
||||
this.kafkaStreamsBindingInformationCatalogue.addConsumerPropertiesPerSbfb(streamsBuilderFactoryBean,
|
||||
bindingServiceProperties.getConsumerProperties(input));
|
||||
arguments[index] = table;
|
||||
}
|
||||
else if (parameterType.isAssignableFrom(GlobalKTable.class)) {
|
||||
String materializedAs = extendedConsumerProperties.getMaterializedAs();
|
||||
String bindingDestination = this.bindingServiceProperties.getBindingDestination(input);
|
||||
GlobalKTable<?, ?> table = getGlobalKTable(extendedConsumerProperties, streamsBuilder, keySerde, valueSerde, materializedAs,
|
||||
bindingDestination, autoOffsetReset);
|
||||
GlobalKTableBoundElementFactory.GlobalKTableWrapper globalKTableWrapper =
|
||||
(GlobalKTableBoundElementFactory.GlobalKTableWrapper) targetBean;
|
||||
//wrap the proxy created during the initial target type binding with real object (KTable)
|
||||
globalKTableWrapper.wrap((GlobalKTable<Object, Object>) table);
|
||||
this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding(input, streamsBuilderFactoryBean);
|
||||
this.kafkaStreamsBindingInformationCatalogue.addConsumerPropertiesPerSbfb(streamsBuilderFactoryBean,
|
||||
bindingServiceProperties.getConsumerProperties(input));
|
||||
arguments[index] = table;
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings({ "unchecked" })
|
||||
protected StreamsBuilderFactoryBean buildStreamsBuilderAndRetrieveConfig(String beanNamePostPrefix,
|
||||
ApplicationContext applicationContext, String inboundName,
|
||||
KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties,
|
||||
StreamsBuilderFactoryBeanConfigurer customizer,
|
||||
ConfigurableEnvironment environment, BindingProperties bindingProperties) {
|
||||
ConfigurableListableBeanFactory beanFactory = this.applicationContext
|
||||
.getBeanFactory();
|
||||
|
||||
Map<String, Object> streamConfigGlobalProperties = applicationContext
|
||||
.getBean("streamConfigGlobalProperties", Map.class);
|
||||
|
||||
// Use a copy because the global configuration will be shared by multiple processors.
|
||||
Map<String, Object> streamConfiguration = new HashMap<>(streamConfigGlobalProperties);
|
||||
|
||||
if (kafkaStreamsBinderConfigurationProperties != null) {
|
||||
final Map<String, KafkaStreamsBinderConfigurationProperties.Functions> functionConfigMap = kafkaStreamsBinderConfigurationProperties.getFunctions();
|
||||
if (!CollectionUtils.isEmpty(functionConfigMap)) {
|
||||
final KafkaStreamsBinderConfigurationProperties.Functions functionConfig = functionConfigMap.get(beanNamePostPrefix);
|
||||
if (functionConfig != null) {
|
||||
final Map<String, String> functionSpecificConfig = functionConfig.getConfiguration();
|
||||
if (!CollectionUtils.isEmpty(functionSpecificConfig)) {
|
||||
streamConfiguration.putAll(functionSpecificConfig);
|
||||
}
|
||||
|
||||
String applicationId = functionConfig.getApplicationId();
|
||||
if (StringUtils.hasText(applicationId)) {
|
||||
streamConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
final MutablePropertySources propertySources = environment.getPropertySources();
|
||||
|
||||
if (StringUtils.hasText(bindingProperties.getBinder())) {
|
||||
final KafkaStreamsBinderConfigurationProperties multiBinderKafkaStreamsBinderConfigurationProperties =
|
||||
applicationContext.getBean(bindingProperties.getBinder() + "-KafkaStreamsBinderConfigurationProperties", KafkaStreamsBinderConfigurationProperties.class);
|
||||
String connectionString = multiBinderKafkaStreamsBinderConfigurationProperties.getKafkaConnectionString();
|
||||
if (!StringUtils.hasText(connectionString)) {
|
||||
connectionString = (String) propertySources.get(bindingProperties.getBinder() + "-kafkaStreamsBinderEnv").getProperty("spring.cloud.stream.kafka.binder.brokers");
|
||||
}
|
||||
|
||||
streamConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, connectionString);
|
||||
|
||||
String binderProvidedApplicationId = multiBinderKafkaStreamsBinderConfigurationProperties.getApplicationId();
|
||||
if (StringUtils.hasText(binderProvidedApplicationId)) {
|
||||
streamConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG,
|
||||
binderProvidedApplicationId);
|
||||
}
|
||||
|
||||
if (multiBinderKafkaStreamsBinderConfigurationProperties
|
||||
.getDeserializationExceptionHandler() == DeserializationExceptionHandler.logAndContinue) {
|
||||
streamConfiguration.put(
|
||||
StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG,
|
||||
LogAndContinueExceptionHandler.class);
|
||||
}
|
||||
else if (multiBinderKafkaStreamsBinderConfigurationProperties
|
||||
.getDeserializationExceptionHandler() == DeserializationExceptionHandler.logAndFail) {
|
||||
streamConfiguration.put(
|
||||
StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG,
|
||||
LogAndFailExceptionHandler.class);
|
||||
}
|
||||
else if (multiBinderKafkaStreamsBinderConfigurationProperties
|
||||
.getDeserializationExceptionHandler() == DeserializationExceptionHandler.sendToDlq) {
|
||||
streamConfiguration.put(
|
||||
StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG,
|
||||
RecoveringDeserializationExceptionHandler.class);
|
||||
SendToDlqAndContinue sendToDlqAndContinue = applicationContext.getBean(SendToDlqAndContinue.class);
|
||||
streamConfiguration.put(RecoveringDeserializationExceptionHandler.KSTREAM_DESERIALIZATION_RECOVERER, sendToDlqAndContinue);
|
||||
}
|
||||
|
||||
if (!ObjectUtils.isEmpty(multiBinderKafkaStreamsBinderConfigurationProperties.getConfiguration())) {
|
||||
streamConfiguration.putAll(multiBinderKafkaStreamsBinderConfigurationProperties.getConfiguration());
|
||||
}
|
||||
if (!streamConfiguration.containsKey(StreamsConfig.REPLICATION_FACTOR_CONFIG)) {
|
||||
streamConfiguration.put(StreamsConfig.REPLICATION_FACTOR_CONFIG,
|
||||
(int) multiBinderKafkaStreamsBinderConfigurationProperties.getReplicationFactor());
|
||||
}
|
||||
}
|
||||
|
||||
//this is only used primarily for StreamListener based processors. Although in theory, functions can use it,
|
||||
//it is ideal for functions to use the approach used in the above if statement by using a property like
|
||||
//spring.cloud.stream.kafka.streams.binder.functions.process.configuration.num.threads (assuming that process is the function name).
|
||||
KafkaStreamsConsumerProperties extendedConsumerProperties = this.kafkaStreamsExtendedBindingProperties
|
||||
.getExtendedConsumerProperties(inboundName);
|
||||
Map<String, String> bindingConfig = extendedConsumerProperties.getConfiguration();
|
||||
Assert.state(!bindingConfig.containsKey(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG),
|
||||
ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG + " cannot be overridden at the binding level; "
|
||||
+ "use multiple binders instead");
|
||||
// We will only add the per binding configuration to the current streamConfiguration and not the global one.
|
||||
streamConfiguration
|
||||
.putAll(bindingConfig);
|
||||
|
||||
String bindingLevelApplicationId = extendedConsumerProperties.getApplicationId();
|
||||
// override application.id if set at the individual binding level.
|
||||
// We provide this for backward compatibility with StreamListener based processors.
|
||||
// For function based processors see the approach used above
|
||||
// (i.e. use a property like spring.cloud.stream.kafka.streams.binder.functions.process.applicationId).
|
||||
if (StringUtils.hasText(bindingLevelApplicationId)) {
|
||||
streamConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG,
|
||||
bindingLevelApplicationId);
|
||||
}
|
||||
|
||||
//If the application id is not set by any mechanism, then generate it.
|
||||
streamConfiguration.computeIfAbsent(StreamsConfig.APPLICATION_ID_CONFIG,
|
||||
k -> {
|
||||
String generatedApplicationID = beanNamePostPrefix + "-applicationId";
|
||||
LOG.info("Binder Generated Kafka Streams Application ID: " + generatedApplicationID);
|
||||
LOG.info("Use the binder generated application ID only for development and testing. ");
|
||||
LOG.info("For production deployments, please consider explicitly setting an application ID using a configuration property.");
|
||||
LOG.info("The generated applicationID is static and will be preserved over application restarts.");
|
||||
return generatedApplicationID;
|
||||
});
|
||||
|
||||
handleConcurrency(applicationContext, inboundName, streamConfiguration);
|
||||
|
||||
// Override deserialization exception handlers per binding
|
||||
final DeserializationExceptionHandler deserializationExceptionHandler =
|
||||
extendedConsumerProperties.getDeserializationExceptionHandler();
|
||||
if (deserializationExceptionHandler == DeserializationExceptionHandler.logAndFail) {
|
||||
streamConfiguration.put(
|
||||
StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG,
|
||||
LogAndFailExceptionHandler.class);
|
||||
}
|
||||
else if (deserializationExceptionHandler == DeserializationExceptionHandler.logAndContinue) {
|
||||
streamConfiguration.put(
|
||||
StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG,
|
||||
LogAndContinueExceptionHandler.class);
|
||||
}
|
||||
else if (deserializationExceptionHandler == DeserializationExceptionHandler.sendToDlq) {
|
||||
streamConfiguration.put(
|
||||
StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG,
|
||||
RecoveringDeserializationExceptionHandler.class);
|
||||
streamConfiguration.put(RecoveringDeserializationExceptionHandler.KSTREAM_DESERIALIZATION_RECOVERER,
|
||||
applicationContext.getBean(SendToDlqAndContinue.class));
|
||||
}
|
||||
else if (deserializationExceptionHandler == DeserializationExceptionHandler.skipAndContinue) {
|
||||
streamConfiguration.put(
|
||||
StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG,
|
||||
SkipAndContinueExceptionHandler.class);
|
||||
}
|
||||
|
||||
KafkaStreamsConfiguration kafkaStreamsConfiguration = new KafkaStreamsConfiguration(streamConfiguration);
|
||||
|
||||
StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.cleanupConfig == null
|
||||
? new StreamsBuilderFactoryBean(kafkaStreamsConfiguration)
|
||||
: new StreamsBuilderFactoryBean(kafkaStreamsConfiguration,
|
||||
this.cleanupConfig);
|
||||
|
||||
streamsBuilderFactoryBean.setAutoStartup(false);
|
||||
BeanDefinition streamsBuilderBeanDefinition = BeanDefinitionBuilder
|
||||
.genericBeanDefinition(
|
||||
(Class<StreamsBuilderFactoryBean>) streamsBuilderFactoryBean.getClass(),
|
||||
() -> streamsBuilderFactoryBean)
|
||||
.getRawBeanDefinition();
|
||||
((BeanDefinitionRegistry) beanFactory).registerBeanDefinition(
|
||||
"stream-builder-" + beanNamePostPrefix, streamsBuilderBeanDefinition);
|
||||
|
||||
extendedConsumerProperties.setApplicationId((String) streamConfiguration.get(StreamsConfig.APPLICATION_ID_CONFIG));
|
||||
|
||||
final StreamsBuilderFactoryBean streamsBuilderFactoryBeanFromContext = applicationContext.getBean(
|
||||
"&stream-builder-" + beanNamePostPrefix, StreamsBuilderFactoryBean.class);
|
||||
//At this point, the StreamsBuilderFactoryBean is created. If the users call, getObject()
|
||||
//in the customizer, that should grant access to the StreamsBuilder.
|
||||
if (customizer != null) {
|
||||
customizer.configure(streamsBuilderFactoryBean);
|
||||
}
|
||||
return streamsBuilderFactoryBeanFromContext;
|
||||
}
|
||||
|
||||
private void handleConcurrency(ApplicationContext applicationContext, String inboundName,
|
||||
Map<String, Object> streamConfiguration) {
|
||||
// This rebinding is necessary to capture the concurrency explicitly set by the application.
|
||||
// This is added to fix this issue: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/899
|
||||
org.springframework.boot.context.properties.bind.Binder explicitConcurrencyResolver =
|
||||
new org.springframework.boot.context.properties.bind.Binder(ConfigurationPropertySources.get(applicationContext.getEnvironment()),
|
||||
new PropertySourcesPlaceholdersResolver(applicationContext.getEnvironment()),
|
||||
IntegrationUtils.getConversionService(this.applicationContext.getBeanFactory()), null);
|
||||
|
||||
boolean[] concurrencyExplicitlyProvided = new boolean[] {false};
|
||||
BindHandler handler = new BindHandler() {
|
||||
|
||||
@Override
|
||||
public Object onSuccess(ConfigurationPropertyName name, Bindable<?> target,
|
||||
BindContext context, Object result) {
|
||||
if (!concurrencyExplicitlyProvided[0]) {
|
||||
concurrencyExplicitlyProvided[0] = name.getLastElement(ConfigurationPropertyName.Form.UNIFORM).equals("concurrency") &&
|
||||
// name is normalized to contain only uniform elements and thus safe to call toLowerCase here.
|
||||
ConfigurationPropertyName.of("spring.cloud.stream.bindings." + inboundName.toLowerCase() + ".consumer")
|
||||
.isAncestorOf(name);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
};
|
||||
//Re-bind spring.cloud.stream properties to check if the application explicitly provided concurrency.
|
||||
try {
|
||||
explicitConcurrencyResolver.bind("spring.cloud.stream",
|
||||
Bindable.ofInstance(new BindingServiceProperties()), handler);
|
||||
}
|
||||
catch (Exception e) {
|
||||
// Ignore this exception
|
||||
}
|
||||
|
||||
int concurrency = this.bindingServiceProperties.getConsumerProperties(inboundName)
|
||||
.getConcurrency();
|
||||
// override concurrency if set at the individual binding level.
|
||||
// Concurrency will be mapped to num.stream.threads.
|
||||
// This conditional also takes into account explicit concurrency settings left at the default value of 1
|
||||
// by the application to address concurrency behavior in applications with multiple processors.
|
||||
// See this GH issue: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/844
|
||||
if (concurrency >= 1 && concurrencyExplicitlyProvided[0]) {
|
||||
streamConfiguration.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG,
|
||||
concurrency);
|
||||
}
|
||||
}
|
||||
|
||||
protected Serde<?> getValueSerde(String inboundName, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, ResolvableType resolvableType) {
|
||||
if (bindingServiceProperties.getConsumerProperties(inboundName).isUseNativeDecoding()) {
|
||||
BindingProperties bindingProperties = this.bindingServiceProperties
|
||||
.getBindingProperties(inboundName);
|
||||
return this.keyValueSerdeResolver.getInboundValueSerde(
|
||||
bindingProperties.getConsumer(), kafkaStreamsConsumerProperties, resolvableType);
|
||||
}
|
||||
else {
|
||||
return Serdes.ByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked"})
|
||||
protected KStream<?, ?> getKStream(String inboundName, BindingProperties bindingProperties, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties,
|
||||
StreamsBuilder streamsBuilder, Serde<?> keySerde, Serde<?> valueSerde, Topology.AutoOffsetReset autoOffsetReset, boolean firstBuild) {
|
||||
if (firstBuild) {
|
||||
addStateStoreBeans(streamsBuilder);
|
||||
}
|
||||
|
||||
final boolean nativeDecoding = this.bindingServiceProperties
|
||||
.getConsumerProperties(inboundName).isUseNativeDecoding();
|
||||
if (nativeDecoding) {
|
||||
LOG.info("Native decoding is enabled for " + inboundName
|
||||
+ ". Inbound deserialization done at the broker.");
|
||||
}
|
||||
else {
|
||||
LOG.info("Native decoding is disabled for " + inboundName
|
||||
+ ". Inbound message conversion done by Spring Cloud Stream.");
|
||||
}
|
||||
|
||||
KStream<?, ?> stream;
|
||||
final Serde<?> valueSerdeToUse = getValueSerdeToUse(kafkaStreamsConsumerProperties, valueSerde);
|
||||
final Consumed<?, ?> consumed = getConsumed(kafkaStreamsConsumerProperties, keySerde, valueSerdeToUse, autoOffsetReset);
|
||||
|
||||
if (this.kafkaStreamsExtendedBindingProperties
|
||||
.getExtendedConsumerProperties(inboundName).isDestinationIsPattern()) {
|
||||
final Pattern pattern = Pattern.compile(this.bindingServiceProperties.getBindingDestination(inboundName));
|
||||
|
||||
stream = streamsBuilder.stream(pattern, consumed);
|
||||
}
|
||||
else {
|
||||
String[] bindingTargets = StringUtils.commaDelimitedListToStringArray(
|
||||
this.bindingServiceProperties.getBindingDestination(inboundName));
|
||||
stream = streamsBuilder.stream(Arrays.asList(bindingTargets),
|
||||
consumed);
|
||||
}
|
||||
//Check to see if event type based routing is enabled.
|
||||
//See this issue for more context: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1003
|
||||
if (StringUtils.hasText(kafkaStreamsConsumerProperties.getEventTypes())) {
|
||||
AtomicBoolean matched = new AtomicBoolean();
|
||||
AtomicReference<String> topicObject = new AtomicReference<>();
|
||||
AtomicReference<Headers> headersObject = new AtomicReference<>();
|
||||
// Processor to retrieve the header value.
|
||||
stream.process(() -> eventTypeProcessor(kafkaStreamsConsumerProperties, matched, topicObject, headersObject));
|
||||
// Branching based on event type match.
|
||||
final KStream<?, ?>[] branch = stream.branch((key, value) -> matched.getAndSet(false));
|
||||
// Deserialize if we have a branch from above.
|
||||
final KStream<?, Object> deserializedKStream = !kafkaStreamsConsumerProperties.isUseConfiguredSerdeWhenRoutingEvents() ?
|
||||
branch[0].mapValues(value -> valueSerde.deserializer().deserialize(
|
||||
topicObject.get(), headersObject.get(), ((Bytes) value).get())) : (KStream<?, Object>) branch[0];
|
||||
return getkStream(bindingProperties, deserializedKStream, nativeDecoding);
|
||||
}
|
||||
return getkStream(bindingProperties, stream, nativeDecoding);
|
||||
}
|
||||
|
||||
private Serde<?> getValueSerdeToUse(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, Serde<?> valueSerde) {
|
||||
if (StringUtils.hasText(kafkaStreamsConsumerProperties.getEventTypes())) {
|
||||
return kafkaStreamsConsumerProperties.isUseConfiguredSerdeWhenRoutingEvents() ? valueSerde : new Serdes.BytesSerde();
|
||||
}
|
||||
return valueSerde;
|
||||
}
|
||||
|
||||
private KStream<?, ?> getkStream(BindingProperties bindingProperties, KStream<?, ?> stream, boolean nativeDecoding) {
|
||||
if (!nativeDecoding) {
|
||||
AtomicReference<Headers> headersAtomicReference = new AtomicReference<>();
|
||||
stream.process((ProcessorSupplier<Object, Object, Void, Void>) () -> new Processor<Object, Object, Void, Void>() {
|
||||
|
||||
@Override
|
||||
public void init(ProcessorContext<Void, Void> context) {
|
||||
Processor.super.init(context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Record<Object, Object> record) {
|
||||
final Headers headers = record.headers();
|
||||
headersAtomicReference.set(headers);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
Processor.super.close();
|
||||
}
|
||||
});
|
||||
stream = stream.mapValues((value) -> {
|
||||
Object returnValue;
|
||||
String contentType = bindingProperties.getContentType();
|
||||
if (value != null && StringUtils.hasText(contentType)) {
|
||||
final Headers headers = headersAtomicReference.get();
|
||||
final Map<String, Object> headersMap = new HashMap<>();
|
||||
headers.forEach(header -> headersMap.put(header.key(), header.value()));
|
||||
returnValue = MessageBuilder.withPayload(value).copyHeaders(headersMap)
|
||||
.setHeader(MessageHeaders.CONTENT_TYPE, contentType).build();
|
||||
}
|
||||
else {
|
||||
returnValue = value;
|
||||
}
|
||||
return returnValue;
|
||||
});
|
||||
}
|
||||
return stream;
|
||||
}
|
||||
|
||||
@SuppressWarnings("rawtypes")
|
||||
private void addStateStoreBeans(StreamsBuilder streamsBuilder) {
|
||||
try {
|
||||
final Map<String, StoreBuilder> storeBuilders = applicationContext.getBeansOfType(StoreBuilder.class);
|
||||
if (!CollectionUtils.isEmpty(storeBuilders)) {
|
||||
storeBuilders.values().forEach(storeBuilder -> {
|
||||
streamsBuilder.addStateStore(storeBuilder);
|
||||
if (LOG.isInfoEnabled()) {
|
||||
LOG.info("state store " + storeBuilder.name() + " added to topology");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
// Pass through.
|
||||
}
|
||||
}
|
||||
|
||||
private <K, V> KTable<K, V> materializedAs(StreamsBuilder streamsBuilder, String destination, String storeName,
|
||||
Serde<K> k, Serde<V> v, Topology.AutoOffsetReset autoOffsetReset, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties) {
|
||||
|
||||
final Consumed<K, V> consumed = getConsumed(kafkaStreamsConsumerProperties, k, v, autoOffsetReset);
|
||||
return streamsBuilder.table(this.bindingServiceProperties.getBindingDestination(destination),
|
||||
consumed, getMaterialized(storeName, k, v));
|
||||
}
|
||||
|
||||
private <K, V> Materialized<K, V, KeyValueStore<Bytes, byte[]>> getMaterialized(
|
||||
String storeName, Serde<K> k, Serde<V> v) {
|
||||
return Materialized.<K, V, KeyValueStore<Bytes, byte[]>>as(storeName)
|
||||
.withKeySerde(k).withValueSerde(v);
|
||||
}
|
||||
|
||||
private <K, V> GlobalKTable<K, V> materializedAsGlobalKTable(
|
||||
StreamsBuilder streamsBuilder, String destination, String storeName,
|
||||
Serde<K> k, Serde<V> v, Topology.AutoOffsetReset autoOffsetReset, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties) {
|
||||
final Consumed<K, V> consumed = getConsumed(kafkaStreamsConsumerProperties, k, v, autoOffsetReset);
|
||||
return streamsBuilder.globalTable(
|
||||
this.bindingServiceProperties.getBindingDestination(destination),
|
||||
consumed,
|
||||
getMaterialized(storeName, k, v));
|
||||
}
|
||||
|
||||
private GlobalKTable<?, ?> getGlobalKTable(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties,
|
||||
StreamsBuilder streamsBuilder,
|
||||
Serde<?> keySerde, Serde<?> valueSerde, String materializedAs,
|
||||
String bindingDestination, Topology.AutoOffsetReset autoOffsetReset) {
|
||||
final Consumed<?, ?> consumed = getConsumed(kafkaStreamsConsumerProperties, keySerde, valueSerde, autoOffsetReset);
|
||||
return materializedAs != null
|
||||
? materializedAsGlobalKTable(streamsBuilder, bindingDestination,
|
||||
materializedAs, keySerde, valueSerde, autoOffsetReset, kafkaStreamsConsumerProperties)
|
||||
: streamsBuilder.globalTable(bindingDestination,
|
||||
consumed);
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked"})
|
||||
private KTable<?, ?> getKTable(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties,
|
||||
StreamsBuilder streamsBuilder, Serde<?> keySerde,
|
||||
Serde<?> valueSerde, String materializedAs, String bindingDestination,
|
||||
Topology.AutoOffsetReset autoOffsetReset) {
|
||||
|
||||
final Serde<?> valueSerdeToUse = getValueSerdeToUse(kafkaStreamsConsumerProperties, valueSerde);
|
||||
|
||||
final Consumed<?, ?> consumed = getConsumed(kafkaStreamsConsumerProperties, keySerde, valueSerdeToUse, autoOffsetReset);
|
||||
|
||||
final KTable<?, ?> kTable = materializedAs != null
|
||||
? materializedAs(streamsBuilder, bindingDestination, materializedAs,
|
||||
keySerde, valueSerdeToUse, autoOffsetReset, kafkaStreamsConsumerProperties)
|
||||
: streamsBuilder.table(bindingDestination,
|
||||
consumed);
|
||||
if (StringUtils.hasText(kafkaStreamsConsumerProperties.getEventTypes())) {
|
||||
AtomicBoolean matched = new AtomicBoolean();
|
||||
AtomicReference<String> topicObject = new AtomicReference<>();
|
||||
AtomicReference<Headers> headersObject = new AtomicReference<>();
|
||||
|
||||
final KStream<?, ?> stream = kTable.toStream();
|
||||
|
||||
// Processor to retrieve the header value.
|
||||
stream.process(() -> eventTypeProcessor(kafkaStreamsConsumerProperties, matched, topicObject, headersObject));
|
||||
// Branching based on event type match.
|
||||
final Map<String, ? extends KStream<?, ?>> stringKStreamMap = stream.split()
|
||||
.branch((key, value) -> matched.getAndSet(false))
|
||||
.noDefaultBranch();
|
||||
final KStream<?, ?>[] branch = stringKStreamMap.values().toArray(new KStream[0]);
|
||||
// Deserialize if we have a branch from above.
|
||||
final KStream<?, Object> deserializedKStream = !kafkaStreamsConsumerProperties.isUseConfiguredSerdeWhenRoutingEvents() ?
|
||||
branch[0].mapValues(value -> valueSerde.deserializer().deserialize(
|
||||
topicObject.get(), headersObject.get(), ((Bytes) value).get())) : (KStream<?, Object>) branch[0];
|
||||
|
||||
return deserializedKStream.toTable();
|
||||
}
|
||||
return kTable;
|
||||
}
|
||||
|
||||
private <K, V> Consumed<K, V> getConsumed(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties,
|
||||
Serde<K> keySerde, Serde<V> valueSerde, Topology.AutoOffsetReset autoOffsetReset) {
|
||||
TimestampExtractor timestampExtractor = null;
|
||||
if (StringUtils.hasText(kafkaStreamsConsumerProperties.getTimestampExtractorBeanName())) {
|
||||
timestampExtractor = applicationContext.getBean(kafkaStreamsConsumerProperties.getTimestampExtractorBeanName(),
|
||||
TimestampExtractor.class);
|
||||
}
|
||||
final Consumed<K, V> consumed = Consumed.with(keySerde, valueSerde)
|
||||
.withOffsetResetPolicy(autoOffsetReset);
|
||||
if (timestampExtractor != null) {
|
||||
consumed.withTimestampExtractor(timestampExtractor);
|
||||
}
|
||||
if (StringUtils.hasText(kafkaStreamsConsumerProperties.getConsumedAs())) {
|
||||
consumed.withName(kafkaStreamsConsumerProperties.getConsumedAs());
|
||||
}
|
||||
return consumed;
|
||||
}
|
||||
|
||||
private <K, V> Processor<K, V, Void, Void> eventTypeProcessor(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties,
|
||||
AtomicBoolean matched, AtomicReference<String> topicObject, AtomicReference<Headers> headersObject) {
|
||||
return new Processor<>() {
|
||||
|
||||
org.apache.kafka.streams.processor.api.ProcessorContext<?, ?> context;
|
||||
|
||||
@Override
|
||||
public void init(org.apache.kafka.streams.processor.api.ProcessorContext<Void, Void> context) {
|
||||
Processor.super.init(context);
|
||||
this.context = context;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Record<K, V> record) {
|
||||
final Headers headers = record.headers();
|
||||
headersObject.set(headers);
|
||||
final Optional<RecordMetadata> optional = this.context.recordMetadata();
|
||||
if (optional.isPresent()) {
|
||||
final RecordMetadata recordMetadata = optional.get();
|
||||
topicObject.set(recordMetadata.topic());
|
||||
}
|
||||
final Iterable<Header> eventTypeHeader = headers.headers(kafkaStreamsConsumerProperties.getEventTypeHeaderKey());
|
||||
if (eventTypeHeader != null && eventTypeHeader.iterator().hasNext()) {
|
||||
String eventTypeFromHeader = new String(eventTypeHeader.iterator().next().value());
|
||||
final String[] eventTypesFromBinding = StringUtils.commaDelimitedListToStringArray(kafkaStreamsConsumerProperties.getEventTypes());
|
||||
for (String eventTypeFromBinding : eventTypesFromBinding) {
|
||||
if (eventTypeFromHeader.equals(eventTypeFromBinding)) {
|
||||
matched.set(true);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
/*
|
||||
* Copyright 2019-2019 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
/**
|
||||
* Enumeration for various {@link org.apache.kafka.streams.errors.DeserializationExceptionHandler} types.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @since 3.0.0
|
||||
*/
|
||||
public enum DeserializationExceptionHandler {
|
||||
|
||||
/**
|
||||
* Deserialization error handler with log and continue.
|
||||
* See {@link org.apache.kafka.streams.errors.LogAndContinueExceptionHandler}
|
||||
*/
|
||||
logAndContinue,
|
||||
/**
|
||||
* Deserialization error handler with log and fail.
|
||||
* See {@link org.apache.kafka.streams.errors.LogAndFailExceptionHandler}
|
||||
*/
|
||||
logAndFail,
|
||||
/**
|
||||
* Deserialization error handler with DLQ send.
|
||||
* See {@link org.springframework.kafka.streams.RecoveringDeserializationExceptionHandler}
|
||||
*/
|
||||
sendToDlq,
|
||||
/**
|
||||
* Deserialization error handler that silently skips the error and continue.
|
||||
*/
|
||||
skipAndContinue;
|
||||
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
/*
|
||||
* Copyright 2023-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.BiFunction;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.apache.kafka.streams.KeyValue;
|
||||
import org.apache.kafka.streams.processor.api.Processor;
|
||||
import org.apache.kafka.streams.processor.api.ProcessorContext;
|
||||
import org.apache.kafka.streams.processor.api.Record;
|
||||
|
||||
import org.springframework.cloud.stream.function.StreamBridge;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* Custom {@link Processor} implementation that is capable of sending a record
|
||||
* to a DLT if the processing fails.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @since 4.1.0
|
||||
*/
|
||||
public class DltAwareProcessor<KIn, VIn, KOut, VOut> implements Processor<KIn, VIn, KOut, VOut> {
|
||||
|
||||
/**
|
||||
* Delegate {@link BiFunction} that is responsible for processing the data.
|
||||
*/
|
||||
private final BiFunction<KIn, VIn, KeyValue<KOut, VOut>> delegateFunction;
|
||||
|
||||
/**
|
||||
* Event time for the forwarded downstream record.
|
||||
*/
|
||||
private final Supplier<Long> recordTimeSupplier;
|
||||
|
||||
/**
|
||||
* DLT destination.
|
||||
*/
|
||||
private String dltDestination;
|
||||
|
||||
/**
|
||||
* {@link DltPublishingContext} used for DLT publishing needs.
|
||||
*/
|
||||
private DltPublishingContext dltPublishingContext;
|
||||
|
||||
/**
|
||||
* A {@link BiConsumer} that does the recovery of a failed record.
|
||||
*/
|
||||
private BiConsumer<Record<KIn, VIn>, Exception> processorRecordRecoverer;
|
||||
|
||||
/**
|
||||
* {@link ProcessorContext} used in the processor.
|
||||
*/
|
||||
private ProcessorContext<KOut, VOut> context;
|
||||
|
||||
/**
|
||||
*
|
||||
* @param delegateFunction {@link BiFunction} to process the data
|
||||
* @param dltDestination DLT destination
|
||||
* @param dltPublishingContext {@link DltPublishingContext}
|
||||
*/
|
||||
public DltAwareProcessor(BiFunction<KIn, VIn, KeyValue<KOut, VOut>> delegateFunction, String dltDestination,
|
||||
DltPublishingContext dltPublishingContext) {
|
||||
this(delegateFunction, dltDestination, dltPublishingContext, System::currentTimeMillis);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param delegateFunction {@link BiFunction} to process the data
|
||||
* @param dltDestination DLT destination
|
||||
* @param dltPublishingContext {@link DltPublishingContext}
|
||||
* @param recordTimeSupplier Supplier for downstream record timestamp
|
||||
*/
|
||||
public DltAwareProcessor(BiFunction<KIn, VIn, KeyValue<KOut, VOut>> delegateFunction, String dltDestination,
|
||||
DltPublishingContext dltPublishingContext, Supplier<Long> recordTimeSupplier) {
|
||||
this.delegateFunction = delegateFunction;
|
||||
this.recordTimeSupplier = recordTimeSupplier;
|
||||
Assert.isTrue(StringUtils.hasText(dltDestination), "DLT Destination topic must be provided.");
|
||||
this.dltDestination = dltDestination;
|
||||
Assert.notNull(dltPublishingContext, "DltSenderContext cannot be null");
|
||||
this.dltPublishingContext = dltPublishingContext;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param delegateFunction {@link BiFunction} to process the data
|
||||
* @param processorRecordRecoverer {@link BiConsumer} that recovers failed records
|
||||
*/
|
||||
public DltAwareProcessor(BiFunction<KIn, VIn, KeyValue<KOut, VOut>> delegateFunction,
|
||||
BiConsumer<Record<KIn, VIn>, Exception> processorRecordRecoverer) {
|
||||
this(delegateFunction, processorRecordRecoverer, System::currentTimeMillis);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param delegateFunction {@link BiFunction} to process the data
|
||||
* @param processorRecordRecoverer {@link BiConsumer} that recovers failed records
|
||||
* @param recordTimeSupplier Supplier for downstream record timestamp
|
||||
*/
|
||||
public DltAwareProcessor(BiFunction<KIn, VIn, KeyValue<KOut, VOut>> delegateFunction,
|
||||
BiConsumer<Record<KIn, VIn>, Exception> processorRecordRecoverer, Supplier<Long> recordTimeSupplier) {
|
||||
this.delegateFunction = delegateFunction;
|
||||
this.recordTimeSupplier = recordTimeSupplier;
|
||||
Assert.notNull(processorRecordRecoverer, "You must provide a valid processor recoverer");
|
||||
this.processorRecordRecoverer = processorRecordRecoverer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(ProcessorContext<KOut, VOut> context) {
|
||||
Processor.super.init(context);
|
||||
this.context = context;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Record<KIn, VIn> record) {
|
||||
try {
|
||||
KeyValue<KOut, VOut> keyValue = this.delegateFunction.apply(record.key(), record.value());
|
||||
Record<KOut, VOut> downstreamRecord = new Record<>(keyValue.key, keyValue.value, recordTimeSupplier.get(), record.headers());
|
||||
this.context.forward(downstreamRecord);
|
||||
}
|
||||
catch (Exception exception) {
|
||||
if (this.processorRecordRecoverer == null) {
|
||||
this.processorRecordRecoverer = defaultProcessorRecordRecoverer();
|
||||
}
|
||||
this.processorRecordRecoverer.accept(record, exception);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
Processor.super.close();
|
||||
}
|
||||
|
||||
BiConsumer<Record<KIn, VIn>, Exception> defaultProcessorRecordRecoverer() {
|
||||
return (r, e) -> {
|
||||
StreamBridge streamBridge = this.dltPublishingContext.getStreamBridge();
|
||||
if (streamBridge != null) {
|
||||
streamBridge.send(this.dltDestination, r.value());
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
/*
|
||||
* Copyright 2023-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
import org.springframework.cloud.stream.function.StreamBridge;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ApplicationContextAware;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
|
||||
/**
|
||||
* The DltPublishingContext is meant to be used along with {@link DltAwareProcessor}
|
||||
* when publishing failed record to a DLT. DltPublishingContext is particularly used
|
||||
* for accessing framework beans such as {@link StreamBridge}.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
public class DltPublishingContext implements ApplicationContextAware, InitializingBean {
|
||||
|
||||
private ConfigurableApplicationContext applicationContext;
|
||||
|
||||
private StreamBridge streamBridge;
|
||||
|
||||
@Override
|
||||
public void afterPropertiesSet() {
|
||||
this.streamBridge = applicationContext.getBean(StreamBridge.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
|
||||
this.applicationContext = (ConfigurableApplicationContext) applicationContext;
|
||||
}
|
||||
|
||||
public StreamBridge getStreamBridge() {
|
||||
return this.streamBridge;
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
/*
|
||||
* Copyright 2019-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationPropertiesBindHandlerAdvisor;
|
||||
import org.springframework.boot.context.properties.bind.AbstractBindHandler;
|
||||
import org.springframework.boot.context.properties.bind.BindContext;
|
||||
import org.springframework.boot.context.properties.bind.BindHandler;
|
||||
import org.springframework.boot.context.properties.bind.BindResult;
|
||||
import org.springframework.boot.context.properties.bind.Bindable;
|
||||
import org.springframework.boot.context.properties.source.ConfigurationPropertyName;
|
||||
|
||||
/**
|
||||
* {@link ConfigurationPropertiesBindHandlerAdvisor} to detect nativeEncoding/Decoding settings
|
||||
* provided by the application explicitly.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @since 3.0.0
|
||||
*/
|
||||
public class EncodingDecodingBindAdviceHandler implements ConfigurationPropertiesBindHandlerAdvisor {
|
||||
|
||||
private boolean encodingSettingProvided;
|
||||
private boolean decodingSettingProvided;
|
||||
|
||||
public boolean isDecodingSettingProvided() {
|
||||
return this.decodingSettingProvided;
|
||||
}
|
||||
|
||||
public boolean isEncodingSettingProvided() {
|
||||
return this.encodingSettingProvided;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BindHandler apply(BindHandler bindHandler) {
|
||||
return new AbstractBindHandler(bindHandler) {
|
||||
@Override
|
||||
public <T> Bindable<T> onStart(ConfigurationPropertyName name,
|
||||
Bindable<T> target, BindContext context) {
|
||||
final String configName = name.toString();
|
||||
if (configName.contains("use") && configName.contains("native") &&
|
||||
(configName.contains("encoding") || configName.contains("decoding"))) {
|
||||
BindResult<T> result = context.getBinder().bind(name, target);
|
||||
if (result.isBound()) {
|
||||
if (configName.contains("encoding")) {
|
||||
EncodingDecodingBindAdviceHandler.this.encodingSettingProvided = true;
|
||||
}
|
||||
else {
|
||||
EncodingDecodingBindAdviceHandler.this.decodingSettingProvided = true;
|
||||
}
|
||||
return target.withExistingValue(result.get());
|
||||
}
|
||||
}
|
||||
return bindHandler.onStart(name, target, context);
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2021 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.context.properties.source.ConfigurationPropertyName;
|
||||
import org.springframework.cloud.stream.config.BindingHandlerAdvise.MappingsProvider;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* {@link EnableAutoConfiguration Auto-configuration} for extended binding metadata for Kafka Streams.
|
||||
*
|
||||
* @author Chris Bono
|
||||
* @since 3.2
|
||||
*/
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
public class ExtendedBindingHandlerMappingsProviderAutoConfiguration {
|
||||
|
||||
@Bean
|
||||
public MappingsProvider kafkaStreamsExtendedPropertiesDefaultMappingsProvider() {
|
||||
return () -> {
|
||||
Map<ConfigurationPropertyName, ConfigurationPropertyName> mappings = new HashMap<>();
|
||||
mappings.put(
|
||||
ConfigurationPropertyName.of("spring.cloud.stream.kafka.streams"),
|
||||
ConfigurationPropertyName.of("spring.cloud.stream.kafka.streams.default"));
|
||||
mappings.put(
|
||||
ConfigurationPropertyName.of("spring.cloud.stream.kafka.streams.bindings"),
|
||||
ConfigurationPropertyName.of("spring.cloud.stream.kafka.streams.default"));
|
||||
return mappings;
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2021 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import org.apache.kafka.streams.KafkaStreams;
|
||||
import org.apache.kafka.streams.StreamsConfig;
|
||||
import org.apache.kafka.streams.kstream.GlobalKTable;
|
||||
|
||||
import org.springframework.cloud.stream.binder.AbstractBinder;
|
||||
import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider;
|
||||
import org.springframework.cloud.stream.binder.Binding;
|
||||
import org.springframework.cloud.stream.binder.DefaultBinding;
|
||||
import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsProducerProperties;
|
||||
import org.springframework.kafka.config.StreamsBuilderFactoryBean;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* An {@link AbstractBinder} implementation for {@link GlobalKTable}.
|
||||
* <p>
|
||||
* Provides only consumer binding for the bound {@link GlobalKTable}. Output bindings are
|
||||
* not allowed on this binder.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @since 2.1.0
|
||||
*/
|
||||
public class GlobalKTableBinder extends
|
||||
// @checkstyle:off
|
||||
AbstractBinder<GlobalKTable<Object, Object>, ExtendedConsumerProperties<KafkaStreamsConsumerProperties>, ExtendedProducerProperties<KafkaStreamsProducerProperties>>
|
||||
implements
|
||||
ExtendedPropertiesBinder<GlobalKTable<Object, Object>, KafkaStreamsConsumerProperties, KafkaStreamsProducerProperties> {
|
||||
|
||||
// @checkstyle:on
|
||||
private final KafkaStreamsBinderConfigurationProperties binderConfigurationProperties;
|
||||
|
||||
private final KafkaTopicProvisioner kafkaTopicProvisioner;
|
||||
|
||||
private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue;
|
||||
|
||||
// @checkstyle:off
|
||||
private KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties = new KafkaStreamsExtendedBindingProperties();
|
||||
private final KafkaStreamsRegistry kafkaStreamsRegistry;
|
||||
|
||||
// @checkstyle:on
|
||||
|
||||
public GlobalKTableBinder(
|
||||
KafkaStreamsBinderConfigurationProperties binderConfigurationProperties,
|
||||
KafkaTopicProvisioner kafkaTopicProvisioner,
|
||||
KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, KafkaStreamsRegistry kafkaStreamsRegistry) {
|
||||
this.binderConfigurationProperties = binderConfigurationProperties;
|
||||
this.kafkaTopicProvisioner = kafkaTopicProvisioner;
|
||||
this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue;
|
||||
this.kafkaStreamsRegistry = kafkaStreamsRegistry;
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
protected Binding<GlobalKTable<Object, Object>> doBindConsumer(String name,
|
||||
String group, GlobalKTable<Object, Object> inputTarget,
|
||||
ExtendedConsumerProperties<KafkaStreamsConsumerProperties> properties) {
|
||||
if (!StringUtils.hasText(group)) {
|
||||
group = properties.getExtension().getApplicationId();
|
||||
}
|
||||
final RetryTemplate retryTemplate = buildRetryTemplate(properties);
|
||||
|
||||
final String bindingName = this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(inputTarget);
|
||||
final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsBindingInformationCatalogue
|
||||
.getStreamsBuilderFactoryBeanPerBinding().get(bindingName);
|
||||
|
||||
KafkaStreamsBinderUtils.prepareConsumerBinding(name, group,
|
||||
getApplicationContext(), this.kafkaTopicProvisioner,
|
||||
this.binderConfigurationProperties, properties, retryTemplate, getBeanFactory(),
|
||||
this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(inputTarget),
|
||||
this.kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean);
|
||||
|
||||
return new DefaultBinding<GlobalKTable<Object, Object>>(bindingName, group, inputTarget, streamsBuilderFactoryBean) {
|
||||
|
||||
@Override
|
||||
public boolean isInput() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void start() {
|
||||
if (!streamsBuilderFactoryBean.isRunning()) {
|
||||
super.start();
|
||||
GlobalKTableBinder.this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean);
|
||||
//If we cached the previous KafkaStreams object (from a binding stop on the actuator), remove it.
|
||||
//See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165
|
||||
final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG);
|
||||
if (kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().containsKey(applicationId)) {
|
||||
kafkaStreamsBindingInformationCatalogue.removePreviousKafkaStreamsForApplicationId(applicationId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void stop() {
|
||||
if (streamsBuilderFactoryBean.isRunning()) {
|
||||
final KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams();
|
||||
super.stop();
|
||||
GlobalKTableBinder.this.kafkaStreamsRegistry.unregisterKafkaStreams(kafkaStreams);
|
||||
KafkaStreamsBinderUtils.closeDlqProducerFactories(kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean);
|
||||
//Caching the stopped KafkaStreams for health indicator purposes on the underlying processor.
|
||||
//See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165
|
||||
GlobalKTableBinder.this.kafkaStreamsBindingInformationCatalogue.addPreviousKafkaStreamsForApplicationId(
|
||||
(String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG), kafkaStreams);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Binding<GlobalKTable<Object, Object>> doBindProducer(String name,
|
||||
GlobalKTable<Object, Object> outboundBindTarget,
|
||||
ExtendedProducerProperties<KafkaStreamsProducerProperties> properties) {
|
||||
throw new UnsupportedOperationException(
|
||||
"No producer level binding is allowed for GlobalKTable");
|
||||
}
|
||||
|
||||
@Override
|
||||
public KafkaStreamsConsumerProperties getExtendedConsumerProperties(
|
||||
String channelName) {
|
||||
return this.kafkaStreamsExtendedBindingProperties
|
||||
.getExtendedConsumerProperties(channelName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public KafkaStreamsProducerProperties getExtendedProducerProperties(
|
||||
String channelName) {
|
||||
throw new UnsupportedOperationException(
|
||||
"No producer binding is allowed and therefore no properties");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDefaultsPrefix() {
|
||||
return this.kafkaStreamsExtendedBindingProperties.getDefaultsPrefix();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<? extends BinderSpecificPropertiesProvider> getExtendedPropertiesEntryClass() {
|
||||
return this.kafkaStreamsExtendedBindingProperties
|
||||
.getExtendedPropertiesEntryClass();
|
||||
}
|
||||
|
||||
public void setKafkaStreamsExtendedBindingProperties(
|
||||
KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties) {
|
||||
this.kafkaStreamsExtendedBindingProperties = kafkaStreamsExtendedBindingProperties;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
|
||||
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.AdminClientConfigCustomizer;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
/**
|
||||
* Configuration for GlobalKTable binder.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @author Chris Bono
|
||||
* @since 2.1.0
|
||||
*/
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@Import({ MultiBinderPropertiesConfiguration.class,
|
||||
KafkaStreamsBinderHealthIndicatorConfiguration.class,
|
||||
KafkaStreamsJaasConfiguration.class})
|
||||
@EnableConfigurationProperties(KafkaProperties.class)
|
||||
public class GlobalKTableBinderConfiguration {
|
||||
|
||||
@Bean
|
||||
public KafkaTopicProvisioner provisioningProvider(
|
||||
KafkaStreamsBinderConfigurationProperties binderConfigurationProperties,
|
||||
KafkaProperties kafkaProperties, ObjectProvider<AdminClientConfigCustomizer> adminClientConfigCustomizer) {
|
||||
return new KafkaTopicProvisioner(binderConfigurationProperties, kafkaProperties, adminClientConfigCustomizer.getIfUnique());
|
||||
}
|
||||
|
||||
@Bean
|
||||
public GlobalKTableBinder GlobalKTableBinder(
|
||||
KafkaStreamsBinderConfigurationProperties binderConfigurationProperties,
|
||||
KafkaTopicProvisioner kafkaTopicProvisioner,
|
||||
KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties,
|
||||
KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue,
|
||||
@Qualifier("streamConfigGlobalProperties") Map<String, Object> streamConfigGlobalProperties,
|
||||
KafkaStreamsRegistry kafkaStreamsRegistry) {
|
||||
|
||||
GlobalKTableBinder globalKTableBinder = new GlobalKTableBinder(binderConfigurationProperties,
|
||||
kafkaTopicProvisioner, kafkaStreamsBindingInformationCatalogue, kafkaStreamsRegistry);
|
||||
globalKTableBinder.setKafkaStreamsExtendedBindingProperties(
|
||||
kafkaStreamsExtendedBindingProperties);
|
||||
return globalKTableBinder;
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnBean(name = "outerContext")
|
||||
public static BeanFactoryPostProcessor outerContextBeanFactoryPostProcessor() {
|
||||
return beanFactory -> {
|
||||
|
||||
// It is safe to call getBean("outerContext") here, because this bean is
|
||||
// registered as first
|
||||
// and as independent from the parent context.
|
||||
ApplicationContext outerContext = (ApplicationContext) beanFactory
|
||||
.getBean("outerContext");
|
||||
beanFactory.registerSingleton(
|
||||
KafkaStreamsExtendedBindingProperties.class.getSimpleName(),
|
||||
outerContext.getBean(KafkaStreamsExtendedBindingProperties.class));
|
||||
beanFactory.registerSingleton(
|
||||
KafkaStreamsBindingInformationCatalogue.class.getSimpleName(),
|
||||
outerContext.getBean(KafkaStreamsBindingInformationCatalogue.class));
|
||||
beanFactory.registerSingleton(
|
||||
KafkaStreamsRegistry.class.getSimpleName(),
|
||||
outerContext.getBean(KafkaStreamsRegistry.class));
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import org.aopalliance.intercept.MethodInterceptor;
|
||||
import org.aopalliance.intercept.MethodInvocation;
|
||||
import org.apache.kafka.streams.kstream.GlobalKTable;
|
||||
|
||||
import org.springframework.aop.framework.ProxyFactory;
|
||||
import org.springframework.cloud.stream.binder.ConsumerProperties;
|
||||
import org.springframework.cloud.stream.binding.AbstractBindingTargetFactory;
|
||||
import org.springframework.cloud.stream.config.BindingProperties;
|
||||
import org.springframework.cloud.stream.config.BindingServiceProperties;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
/**
|
||||
* {@link org.springframework.cloud.stream.binding.BindingTargetFactory} for
|
||||
* {@link GlobalKTable}
|
||||
*
|
||||
* Input bindings are only created as output bindings on GlobalKTable are not allowed.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @since 2.1.0
|
||||
*/
|
||||
public class GlobalKTableBoundElementFactory
|
||||
extends AbstractBindingTargetFactory<GlobalKTable> {
|
||||
|
||||
private final BindingServiceProperties bindingServiceProperties;
|
||||
private final EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler;
|
||||
private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue;
|
||||
|
||||
GlobalKTableBoundElementFactory(BindingServiceProperties bindingServiceProperties,
|
||||
EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler,
|
||||
KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue) {
|
||||
super(GlobalKTable.class);
|
||||
this.bindingServiceProperties = bindingServiceProperties;
|
||||
this.encodingDecodingBindAdviceHandler = encodingDecodingBindAdviceHandler;
|
||||
this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GlobalKTable createInput(String name) {
|
||||
BindingProperties bindingProperties = this.bindingServiceProperties.getBindingProperties(name);
|
||||
ConsumerProperties consumerProperties = bindingProperties.getConsumer();
|
||||
if (consumerProperties == null) {
|
||||
consumerProperties = this.bindingServiceProperties.getConsumerProperties(name);
|
||||
consumerProperties.setUseNativeDecoding(true);
|
||||
}
|
||||
else {
|
||||
if (!encodingDecodingBindAdviceHandler.isDecodingSettingProvided()) {
|
||||
consumerProperties.setUseNativeDecoding(true);
|
||||
}
|
||||
}
|
||||
// Always set multiplex to true in the kafka streams binder
|
||||
consumerProperties.setMultiplex(true);
|
||||
|
||||
// @checkstyle:off
|
||||
GlobalKTableBoundElementFactory.GlobalKTableWrapperHandler wrapper = new GlobalKTableBoundElementFactory.GlobalKTableWrapperHandler();
|
||||
// @checkstyle:on
|
||||
ProxyFactory proxyFactory = new ProxyFactory(
|
||||
GlobalKTableBoundElementFactory.GlobalKTableWrapper.class,
|
||||
GlobalKTable.class);
|
||||
proxyFactory.addAdvice(wrapper);
|
||||
|
||||
final GlobalKTable proxy = (GlobalKTable) proxyFactory.getProxy();
|
||||
this.kafkaStreamsBindingInformationCatalogue.addBindingNamePerTarget(proxy, name);
|
||||
return proxy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GlobalKTable createOutput(String name) {
|
||||
throw new UnsupportedOperationException(
|
||||
"Outbound operations are not allowed on target type GlobalKTable");
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrapper for GlobalKTable proxy.
|
||||
*/
|
||||
public interface GlobalKTableWrapper {
|
||||
|
||||
void wrap(GlobalKTable<Object, Object> delegate);
|
||||
|
||||
}
|
||||
|
||||
private static class GlobalKTableWrapperHandler implements
|
||||
GlobalKTableBoundElementFactory.GlobalKTableWrapper, MethodInterceptor {
|
||||
|
||||
private GlobalKTable<Object, Object> delegate;
|
||||
|
||||
public void wrap(GlobalKTable<Object, Object> delegate) {
|
||||
Assert.notNull(delegate, "delegate cannot be null");
|
||||
if (this.delegate == null) {
|
||||
this.delegate = delegate;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object invoke(MethodInvocation methodInvocation) throws Throwable {
|
||||
if (methodInvocation.getMethod().getDeclaringClass()
|
||||
.equals(GlobalKTable.class)) {
|
||||
Assert.notNull(this.delegate,
|
||||
"Trying to prepareConsumerBinding " + methodInvocation.getMethod()
|
||||
+ " but no delegate has been set.");
|
||||
return methodInvocation.getMethod().invoke(this.delegate,
|
||||
methodInvocation.getArguments());
|
||||
}
|
||||
else if (methodInvocation.getMethod().getDeclaringClass()
|
||||
.equals(GlobalKTableBoundElementFactory.GlobalKTableWrapper.class)) {
|
||||
return methodInvocation.getMethod().invoke(this,
|
||||
methodInvocation.getArguments());
|
||||
}
|
||||
else {
|
||||
throw new IllegalStateException(
|
||||
"Only GlobalKTable method invocations are permitted");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,322 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.kafka.common.serialization.Serializer;
|
||||
import org.apache.kafka.streams.KafkaStreams;
|
||||
import org.apache.kafka.streams.KeyQueryMetadata;
|
||||
import org.apache.kafka.streams.StoreQueryParameters;
|
||||
import org.apache.kafka.streams.StreamsConfig;
|
||||
import org.apache.kafka.streams.errors.InvalidStateStoreException;
|
||||
import org.apache.kafka.streams.errors.UnknownStateStoreException;
|
||||
import org.apache.kafka.streams.state.HostInfo;
|
||||
import org.apache.kafka.streams.state.QueryableStoreType;
|
||||
import org.apache.kafka.streams.state.StreamsMetadata;
|
||||
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.retry.RetryPolicy;
|
||||
import org.springframework.retry.backoff.FixedBackOffPolicy;
|
||||
import org.springframework.retry.policy.SimpleRetryPolicy;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* Services pertinent to the interactive query capabilities of Kafka Streams. This class
|
||||
* provides services such as querying for a particular store, which instance is hosting a
|
||||
* particular store etc. This is part of the public API of the kafka streams binder and
|
||||
* the users can inject this service in their applications to make use of it.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @author Renwei Han
|
||||
* @author Serhii Siryi
|
||||
* @author Nico Pommerening
|
||||
* @author Chris Bono
|
||||
* @since 2.1.0
|
||||
*/
|
||||
public class InteractiveQueryService {
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(InteractiveQueryService.class);
|
||||
|
||||
private final KafkaStreamsRegistry kafkaStreamsRegistry;
|
||||
|
||||
private final KafkaStreamsBinderConfigurationProperties binderConfigurationProperties;
|
||||
|
||||
private final KafkaStreamsVersionAgnosticTopologyInfoFacade topologyInfoFacade;
|
||||
|
||||
private StoreQueryParametersCustomizer<?> storeQueryParametersCustomizer;
|
||||
|
||||
/**
|
||||
* Constructor for InteractiveQueryService.
|
||||
* @param kafkaStreamsRegistry holding {@link KafkaStreamsRegistry}
|
||||
* @param binderConfigurationProperties kafka Streams binder configuration properties
|
||||
*/
|
||||
public InteractiveQueryService(KafkaStreamsRegistry kafkaStreamsRegistry,
|
||||
KafkaStreamsBinderConfigurationProperties binderConfigurationProperties) {
|
||||
this.kafkaStreamsRegistry = kafkaStreamsRegistry;
|
||||
this.binderConfigurationProperties = binderConfigurationProperties;
|
||||
this.topologyInfoFacade = new KafkaStreamsVersionAgnosticTopologyInfoFacade();
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve and return a queryable store by name created in the application.
|
||||
* @param storeName name of the queryable store
|
||||
* @param storeType type of the queryable store
|
||||
* @param <T> generic queryable store
|
||||
* @return queryable store.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public <T> T getQueryableStore(String storeName, QueryableStoreType<T> storeType) {
|
||||
|
||||
KafkaStreams contextSpecificKafkaStreams = getThreadContextSpecificKafkaStreams();
|
||||
|
||||
StoreQueryParameters<T> storeQueryParams = StoreQueryParameters.fromNameAndType(storeName, storeType);
|
||||
if (this.storeQueryParametersCustomizer != null) {
|
||||
storeQueryParams = ((StoreQueryParametersCustomizer<T>) this.storeQueryParametersCustomizer).customize(storeQueryParams);
|
||||
}
|
||||
|
||||
AtomicReference<StoreQueryParameters<T>> storeQueryParametersAtomicReference = new AtomicReference<>(storeQueryParams);
|
||||
|
||||
return getRetryTemplate().execute(context -> {
|
||||
T store = null;
|
||||
Throwable throwable = null;
|
||||
if (contextSpecificKafkaStreams != null) {
|
||||
try {
|
||||
store = contextSpecificKafkaStreams.store(storeQueryParametersAtomicReference.get());
|
||||
}
|
||||
catch (InvalidStateStoreException e) {
|
||||
throwable = e;
|
||||
}
|
||||
}
|
||||
if (store != null) {
|
||||
return store;
|
||||
}
|
||||
if (contextSpecificKafkaStreams != null) {
|
||||
LOG.warn("Store (" + storeName + ") could not be found in Streams context, falling back to all known Streams instances");
|
||||
}
|
||||
|
||||
// Find all apps that know about the store
|
||||
Map<KafkaStreams, T> candidateStores = new HashMap<>();
|
||||
for (KafkaStreams kafkaStreamApp : kafkaStreamsRegistry.getKafkaStreams()) {
|
||||
try {
|
||||
candidateStores.put(kafkaStreamApp, kafkaStreamApp.store(storeQueryParametersAtomicReference.get()));
|
||||
}
|
||||
catch (Exception ex) {
|
||||
throwable = ex;
|
||||
}
|
||||
}
|
||||
|
||||
// Store exists in a single app - no further resolution required
|
||||
if (candidateStores.size() == 1) {
|
||||
return candidateStores.values().stream().findFirst().get();
|
||||
}
|
||||
|
||||
// If the store is in multiple streams apps - discard any apps that do not actually have the store
|
||||
if (candidateStores.size() > 1) {
|
||||
|
||||
candidateStores = candidateStores.entrySet().stream()
|
||||
.filter((e) -> this.topologyInfoFacade.streamsAppActuallyHasStore(e.getKey(), storeName))
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||
|
||||
if (candidateStores.size() == 1) {
|
||||
return candidateStores.values().stream().findFirst().get();
|
||||
}
|
||||
|
||||
throwable = (candidateStores.size() == 0) ?
|
||||
new UnknownStateStoreException("Store (" + storeName + ") not available to Streams instance") :
|
||||
new InvalidStateStoreException("Store (" + storeName + ") available to more than one Streams instance");
|
||||
|
||||
}
|
||||
throw new IllegalStateException("Error retrieving state store: " + storeName, throwable);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the current {@link KafkaStreams} context if executing Thread is created by a Streams App (contains a matching application id in Thread's name).
|
||||
*
|
||||
* @return KafkaStreams instance associated with Thread
|
||||
*/
|
||||
private KafkaStreams getThreadContextSpecificKafkaStreams() {
|
||||
return this.kafkaStreamsRegistry.getKafkaStreams().stream()
|
||||
.filter(this::filterByThreadName).findAny().orElse(null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the supplied {@link KafkaStreams} instance belongs to the calling Thread by matching the Thread's name with the Streams Application Id.
|
||||
*
|
||||
* @param streams {@link KafkaStreams} instance to filter
|
||||
* @return true if Streams Instance is associated with Thread
|
||||
*/
|
||||
private boolean filterByThreadName(KafkaStreams streams) {
|
||||
String applicationId = kafkaStreamsRegistry.streamBuilderFactoryBean(
|
||||
streams).getStreamsConfiguration()
|
||||
.getProperty(StreamsConfig.APPLICATION_ID_CONFIG);
|
||||
// TODO: is there some better way to find out if a Stream App created the Thread?
|
||||
return Thread.currentThread().getName().contains(applicationId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the current {@link HostInfo} that the calling kafka streams application is
|
||||
* running on.
|
||||
*
|
||||
* Note that the end user applications must provide `application.server` as a
|
||||
* configuration property when calling this method. If this is not available, then
|
||||
* null is returned.
|
||||
* @return the current {@link HostInfo}
|
||||
*/
|
||||
public HostInfo getCurrentHostInfo() {
|
||||
Map<String, String> configuration = this.binderConfigurationProperties
|
||||
.getConfiguration();
|
||||
if (configuration.containsKey("application.server")) {
|
||||
|
||||
String applicationServer = configuration.get("application.server");
|
||||
String[] splits = StringUtils.split(applicationServer, ":");
|
||||
|
||||
return new HostInfo(splits[0], Integer.parseInt(splits[1]));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the {@link HostInfo} where the provided store and key are hosted on. This may
|
||||
* not be the current host that is running the application. Kafka Streams will look
|
||||
* through all the consumer instances under the same application id and retrieves the
|
||||
* proper host.
|
||||
*
|
||||
* Note that the end user applications must provide `application.server` as a
|
||||
* configuration property for all the application instances when calling this method.
|
||||
* If this is not available, then null maybe returned.
|
||||
* @param <K> generic type for key
|
||||
* @param store store name
|
||||
* @param key key to look for
|
||||
* @param serializer {@link Serializer} for the key
|
||||
* @return the {@link HostInfo} where the key for the provided store is hosted currently
|
||||
*/
|
||||
public <K> HostInfo getHostInfo(String store, K key, Serializer<K> serializer) {
|
||||
final RetryTemplate retryTemplate = getRetryTemplate();
|
||||
|
||||
|
||||
return retryTemplate.execute(context -> {
|
||||
Throwable throwable = null;
|
||||
try {
|
||||
final KeyQueryMetadata keyQueryMetadata = this.kafkaStreamsRegistry.getKafkaStreams()
|
||||
.stream()
|
||||
.map((k) -> Optional.ofNullable(k.queryMetadataForKey(store, key, serializer)))
|
||||
.filter(Optional::isPresent).map(Optional::get).findFirst().orElse(null);
|
||||
if (keyQueryMetadata != null) {
|
||||
return keyQueryMetadata.activeHost();
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
throwable = e;
|
||||
}
|
||||
throw new IllegalStateException(
|
||||
"Error when retrieving state store.", throwable != null ? throwable : new Throwable("Kafka Streams is not ready."));
|
||||
});
|
||||
}
|
||||
|
||||
private RetryTemplate getRetryTemplate() {
|
||||
RetryTemplate retryTemplate = new RetryTemplate();
|
||||
|
||||
KafkaStreamsBinderConfigurationProperties.StateStoreRetry stateStoreRetry = this.binderConfigurationProperties.getStateStoreRetry();
|
||||
RetryPolicy retryPolicy = new SimpleRetryPolicy(stateStoreRetry.getMaxAttempts());
|
||||
FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy();
|
||||
backOffPolicy.setBackOffPeriod(stateStoreRetry.getBackoffPeriod());
|
||||
|
||||
retryTemplate.setBackOffPolicy(backOffPolicy);
|
||||
retryTemplate.setRetryPolicy(retryPolicy);
|
||||
|
||||
return retryTemplate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves and returns the {@link KeyQueryMetadata} associated with the given combination of
|
||||
* key and state store. If none found, it will return null.
|
||||
*
|
||||
* @param <K> generic type for key
|
||||
* @param store store name
|
||||
* @param key key to look for
|
||||
* @param serializer {@link Serializer} for the key
|
||||
* @return the {@link KeyQueryMetadata} if available, null otherwise.
|
||||
*/
|
||||
public <K> KeyQueryMetadata getKeyQueryMetadata(String store, K key, Serializer<K> serializer) {
|
||||
return this.kafkaStreamsRegistry.getKafkaStreams()
|
||||
.stream()
|
||||
.map((k) -> Optional.ofNullable(k.queryMetadataForKey(store, key, serializer)))
|
||||
.filter(Optional::isPresent).map(Optional::get).findFirst().orElse(null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves and returns the {@link KafkaStreams} object that is associated with the given combination of
|
||||
* key and state store. If none found, it will return null.
|
||||
*
|
||||
* @param <K> generic type for key
|
||||
* @param store store name
|
||||
* @param key key to look for
|
||||
* @param serializer {@link Serializer} for the key
|
||||
* @return {@link KafkaStreams} object associated with this combination of store and key
|
||||
*/
|
||||
public <K> KafkaStreams getKafkaStreams(String store, K key, Serializer<K> serializer) {
|
||||
final AtomicReference<KafkaStreams> kafkaStreamsAtomicReference = new AtomicReference<>();
|
||||
this.kafkaStreamsRegistry.getKafkaStreams()
|
||||
.forEach(k -> {
|
||||
final KeyQueryMetadata keyQueryMetadata = k.queryMetadataForKey(store, key, serializer);
|
||||
if (keyQueryMetadata != null) {
|
||||
kafkaStreamsAtomicReference.set(k);
|
||||
}
|
||||
});
|
||||
return kafkaStreamsAtomicReference.get();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the list of {@link HostInfo} where the provided store is hosted on.
|
||||
* It also can include current host info.
|
||||
* Kafka Streams will look through all the consumer instances under the same application id
|
||||
* and retrieves all hosts info.
|
||||
*
|
||||
* Note that the end-user applications must provide `application.server` as a configuration property
|
||||
* for all the application instances when calling this method. If this is not available, then an empty list will be returned.
|
||||
*
|
||||
* @param store store name
|
||||
* @return the list of {@link HostInfo} where provided store is hosted on
|
||||
*/
|
||||
public List<HostInfo> getAllHostsInfo(String store) {
|
||||
return kafkaStreamsRegistry.getKafkaStreams()
|
||||
.stream()
|
||||
.flatMap(k -> k.allMetadataForStore(store).stream())
|
||||
.filter(Objects::nonNull)
|
||||
.map(StreamsMetadata::hostInfo)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* @param storeQueryParametersCustomizer to customize
|
||||
* @since 4.0.1
|
||||
*/
|
||||
public void setStoreQueryParametersCustomizer(StoreQueryParametersCustomizer storeQueryParametersCustomizer) {
|
||||
this.storeQueryParametersCustomizer = storeQueryParametersCustomizer;
|
||||
}
|
||||
}
|
||||
@@ -1,300 +0,0 @@
|
||||
/*
|
||||
* Copyright 2017-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.kafka.common.serialization.Serde;
|
||||
import org.apache.kafka.common.serialization.Serdes;
|
||||
import org.apache.kafka.streams.KafkaStreams;
|
||||
import org.apache.kafka.streams.StreamsConfig;
|
||||
import org.apache.kafka.streams.kstream.KStream;
|
||||
import org.apache.kafka.streams.kstream.Produced;
|
||||
import org.apache.kafka.streams.processor.StreamPartitioner;
|
||||
|
||||
import org.springframework.aop.framework.Advised;
|
||||
import org.springframework.cloud.stream.binder.AbstractBinder;
|
||||
import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider;
|
||||
import org.springframework.cloud.stream.binder.Binding;
|
||||
import org.springframework.cloud.stream.binder.DefaultBinding;
|
||||
import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsProducerProperties;
|
||||
import org.springframework.kafka.config.StreamsBuilderFactoryBean;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* {@link org.springframework.cloud.stream.binder.Binder} implementation for
|
||||
* {@link KStream}. This implemenation extends from the {@link AbstractBinder} directly.
|
||||
* <p>
|
||||
* Provides both producer and consumer bindings for the bound KStream.
|
||||
*
|
||||
* @author Marius Bogoevici
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
class KStreamBinder extends
|
||||
// @checkstyle:off
|
||||
AbstractBinder<KStream<Object, Object>, ExtendedConsumerProperties<KafkaStreamsConsumerProperties>, ExtendedProducerProperties<KafkaStreamsProducerProperties>>
|
||||
implements
|
||||
ExtendedPropertiesBinder<KStream<Object, Object>, KafkaStreamsConsumerProperties, KafkaStreamsProducerProperties> {
|
||||
|
||||
// @checkstyle:on
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(KStreamBinder.class);
|
||||
|
||||
private final KafkaTopicProvisioner kafkaTopicProvisioner;
|
||||
|
||||
// @checkstyle:off
|
||||
private KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties = new KafkaStreamsExtendedBindingProperties();
|
||||
|
||||
// @checkstyle:on
|
||||
|
||||
private final KafkaStreamsBinderConfigurationProperties binderConfigurationProperties;
|
||||
|
||||
private final KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate;
|
||||
|
||||
private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue;
|
||||
|
||||
private final KeyValueSerdeResolver keyValueSerdeResolver;
|
||||
|
||||
private final KafkaStreamsRegistry kafkaStreamsRegistry;
|
||||
|
||||
KStreamBinder(KafkaStreamsBinderConfigurationProperties binderConfigurationProperties,
|
||||
KafkaTopicProvisioner kafkaTopicProvisioner,
|
||||
KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate,
|
||||
KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue,
|
||||
KeyValueSerdeResolver keyValueSerdeResolver, KafkaStreamsRegistry kafkaStreamsRegistry) {
|
||||
this.binderConfigurationProperties = binderConfigurationProperties;
|
||||
this.kafkaTopicProvisioner = kafkaTopicProvisioner;
|
||||
this.kafkaStreamsMessageConversionDelegate = kafkaStreamsMessageConversionDelegate;
|
||||
this.kafkaStreamsBindingInformationCatalogue = KafkaStreamsBindingInformationCatalogue;
|
||||
this.keyValueSerdeResolver = keyValueSerdeResolver;
|
||||
this.kafkaStreamsRegistry = kafkaStreamsRegistry;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Binding<KStream<Object, Object>> doBindConsumer(String name, String group,
|
||||
KStream<Object, Object> inputTarget,
|
||||
ExtendedConsumerProperties<KafkaStreamsConsumerProperties> properties) {
|
||||
|
||||
KStream<Object, Object> delegate = ((KStreamBoundElementFactory.KStreamWrapperHandler)
|
||||
((Advised) inputTarget).getAdvisors()[0].getAdvice()).getDelegate();
|
||||
|
||||
this.kafkaStreamsBindingInformationCatalogue.registerConsumerProperties(delegate, properties.getExtension());
|
||||
|
||||
if (!StringUtils.hasText(group)) {
|
||||
group = properties.getExtension().getApplicationId();
|
||||
}
|
||||
|
||||
final RetryTemplate retryTemplate = buildRetryTemplate(properties);
|
||||
|
||||
final String bindingName = this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(inputTarget);
|
||||
final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsBindingInformationCatalogue
|
||||
.getStreamsBuilderFactoryBeanPerBinding().get(bindingName);
|
||||
|
||||
KafkaStreamsBinderUtils.prepareConsumerBinding(name, group,
|
||||
getApplicationContext(), this.kafkaTopicProvisioner,
|
||||
this.binderConfigurationProperties, properties, retryTemplate, getBeanFactory(),
|
||||
this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(inputTarget),
|
||||
this.kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean);
|
||||
|
||||
|
||||
return new DefaultBinding<KStream<Object, Object>>(bindingName, group,
|
||||
inputTarget, streamsBuilderFactoryBean) {
|
||||
|
||||
@Override
|
||||
public boolean isInput() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void start() {
|
||||
if (!streamsBuilderFactoryBean.isRunning()) {
|
||||
super.start();
|
||||
KStreamBinder.this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean);
|
||||
//If we cached the previous KafkaStreams object (from a binding stop on the actuator), remove it.
|
||||
//See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165
|
||||
final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG);
|
||||
if (kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().containsKey(applicationId)) {
|
||||
kafkaStreamsBindingInformationCatalogue.removePreviousKafkaStreamsForApplicationId(applicationId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void stop() {
|
||||
if (streamsBuilderFactoryBean.isRunning()) {
|
||||
final KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams();
|
||||
super.stop();
|
||||
KStreamBinder.this.kafkaStreamsRegistry.unregisterKafkaStreams(kafkaStreams);
|
||||
KafkaStreamsBinderUtils.closeDlqProducerFactories(kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean);
|
||||
//Caching the stopped KafkaStreams for health indicator purposes on the underlying processor.
|
||||
//See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165
|
||||
KStreamBinder.this.kafkaStreamsBindingInformationCatalogue.addPreviousKafkaStreamsForApplicationId(
|
||||
(String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG), kafkaStreams);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
protected Binding<KStream<Object, Object>> doBindProducer(String name,
|
||||
KStream<Object, Object> outboundBindTarget,
|
||||
ExtendedProducerProperties<KafkaStreamsProducerProperties> properties) {
|
||||
|
||||
ExtendedProducerProperties<KafkaProducerProperties> extendedProducerProperties =
|
||||
(ExtendedProducerProperties) properties;
|
||||
|
||||
this.kafkaTopicProvisioner.provisionProducerDestination(name, extendedProducerProperties);
|
||||
Serde<?> keySerde = this.keyValueSerdeResolver
|
||||
.getOuboundKeySerde(properties.getExtension(), kafkaStreamsBindingInformationCatalogue.getOutboundKStreamResolvable(outboundBindTarget));
|
||||
LOG.info("Key Serde used for (outbound) " + name + ": " + keySerde.getClass().getName());
|
||||
|
||||
Serde<?> valueSerde;
|
||||
if (properties.isUseNativeEncoding()) {
|
||||
valueSerde = this.keyValueSerdeResolver.getOutboundValueSerde(properties,
|
||||
properties.getExtension(), kafkaStreamsBindingInformationCatalogue.getOutboundKStreamResolvable(outboundBindTarget));
|
||||
}
|
||||
else {
|
||||
valueSerde = Serdes.ByteArray();
|
||||
}
|
||||
LOG.info("Value Serde used for (outbound) " + name + ": " + valueSerde.getClass().getName());
|
||||
|
||||
to(properties.isUseNativeEncoding(), name, outboundBindTarget,
|
||||
(Serde<Object>) keySerde, (Serde<Object>) valueSerde, properties.getExtension());
|
||||
|
||||
final String bindingName = this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(outboundBindTarget);
|
||||
final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsBindingInformationCatalogue
|
||||
.getStreamsBuilderFactoryBeanPerBinding().get(bindingName);
|
||||
|
||||
// We need the application id to pass to DefaultBinding so that it won't be interpreted as an anonymous group.
|
||||
// In case, if we can't find application.id (which is unlikely), we just default to bindingName.
|
||||
// This will only be used for lifecycle management through actuator endpoints.
|
||||
final Properties streamsConfiguration = streamsBuilderFactoryBean.getStreamsConfiguration();
|
||||
final String applicationId = streamsConfiguration != null ? (String) streamsConfiguration.get("application.id") : bindingName;
|
||||
|
||||
return new DefaultBinding<KStream<Object, Object>>(bindingName,
|
||||
applicationId, outboundBindTarget, streamsBuilderFactoryBean) {
|
||||
|
||||
@Override
|
||||
public boolean isInput() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void start() {
|
||||
if (!streamsBuilderFactoryBean.isRunning()) {
|
||||
super.start();
|
||||
KStreamBinder.this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean);
|
||||
//If we cached the previous KafkaStreams object (from a binding stop on the actuator), remove it.
|
||||
//See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165
|
||||
final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG);
|
||||
if (kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().containsKey(applicationId)) {
|
||||
kafkaStreamsBindingInformationCatalogue.removePreviousKafkaStreamsForApplicationId(applicationId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void stop() {
|
||||
if (streamsBuilderFactoryBean.isRunning()) {
|
||||
final KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams();
|
||||
super.stop();
|
||||
KStreamBinder.this.kafkaStreamsRegistry.unregisterKafkaStreams(kafkaStreams);
|
||||
KafkaStreamsBinderUtils.closeDlqProducerFactories(kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean);
|
||||
//Caching the stopped KafkaStreams for health indicator purposes on the underlying processor
|
||||
//See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165
|
||||
KStreamBinder.this.kafkaStreamsBindingInformationCatalogue.addPreviousKafkaStreamsForApplicationId(
|
||||
(String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG), kafkaStreams);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void to(boolean isNativeEncoding, String name,
|
||||
KStream<Object, Object> outboundBindTarget, Serde<Object> keySerde,
|
||||
Serde<Object> valueSerde, KafkaStreamsProducerProperties properties) {
|
||||
final Produced<Object, Object> produced = Produced.with(keySerde, valueSerde);
|
||||
if (StringUtils.hasText(properties.getProducedAs())) {
|
||||
produced.withName(properties.getProducedAs());
|
||||
}
|
||||
StreamPartitioner streamPartitioner = null;
|
||||
if (StringUtils.hasText(properties.getStreamPartitionerBeanName())) {
|
||||
streamPartitioner = getApplicationContext().getBean(properties.getStreamPartitionerBeanName(),
|
||||
StreamPartitioner.class);
|
||||
}
|
||||
if (streamPartitioner != null) {
|
||||
produced.withStreamPartitioner(streamPartitioner);
|
||||
}
|
||||
if (!isNativeEncoding) {
|
||||
LOG.info("Native encoding is disabled for " + name
|
||||
+ ". Outbound message conversion done by Spring Cloud Stream.");
|
||||
outboundBindTarget.filter((k, v) -> v == null)
|
||||
.to(name, produced);
|
||||
this.kafkaStreamsMessageConversionDelegate
|
||||
.serializeOnOutbound(outboundBindTarget)
|
||||
.to(name, produced);
|
||||
}
|
||||
else {
|
||||
LOG.info("Native encoding is enabled for " + name
|
||||
+ ". Outbound serialization done at the broker.");
|
||||
outboundBindTarget.to(name, produced);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public KafkaStreamsConsumerProperties getExtendedConsumerProperties(
|
||||
String channelName) {
|
||||
return this.kafkaStreamsExtendedBindingProperties
|
||||
.getExtendedConsumerProperties(channelName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public KafkaStreamsProducerProperties getExtendedProducerProperties(
|
||||
String channelName) {
|
||||
return this.kafkaStreamsExtendedBindingProperties
|
||||
.getExtendedProducerProperties(channelName);
|
||||
}
|
||||
|
||||
public void setKafkaStreamsExtendedBindingProperties(
|
||||
KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties) {
|
||||
this.kafkaStreamsExtendedBindingProperties = kafkaStreamsExtendedBindingProperties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDefaultsPrefix() {
|
||||
return this.kafkaStreamsExtendedBindingProperties.getDefaultsPrefix();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<? extends BinderSpecificPropertiesProvider> getExtendedPropertiesEntryClass() {
|
||||
return this.kafkaStreamsExtendedBindingProperties
|
||||
.getExtendedPropertiesEntryClass();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
/*
|
||||
* Copyright 2017-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
|
||||
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.AdminClientConfigCustomizer;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
/**
|
||||
* Configuration for KStream binder.
|
||||
*
|
||||
* @author Marius Bogoevici
|
||||
* @author Gary Russell
|
||||
* @author Soby Chacko
|
||||
* @author Chris Bono
|
||||
*/
|
||||
@Configuration
|
||||
@Import({ MultiBinderPropertiesConfiguration.class,
|
||||
KafkaStreamsBinderHealthIndicatorConfiguration.class,
|
||||
KafkaStreamsJaasConfiguration.class})
|
||||
@EnableConfigurationProperties(KafkaProperties.class)
|
||||
public class KStreamBinderConfiguration {
|
||||
|
||||
@Bean
|
||||
public KafkaTopicProvisioner provisioningProvider(
|
||||
KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties,
|
||||
KafkaProperties kafkaProperties, ObjectProvider<AdminClientConfigCustomizer> adminClientConfigCustomizer) {
|
||||
return new KafkaTopicProvisioner(kafkaStreamsBinderConfigurationProperties,
|
||||
kafkaProperties, adminClientConfigCustomizer.getIfUnique());
|
||||
}
|
||||
|
||||
@Bean
|
||||
public KStreamBinder kStreamBinder(
|
||||
KafkaStreamsBinderConfigurationProperties binderConfigurationProperties,
|
||||
KafkaTopicProvisioner kafkaTopicProvisioner,
|
||||
KafkaStreamsMessageConversionDelegate KafkaStreamsMessageConversionDelegate,
|
||||
KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue,
|
||||
KeyValueSerdeResolver keyValueSerdeResolver,
|
||||
KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties, KafkaStreamsRegistry kafkaStreamsRegistry) {
|
||||
KStreamBinder kStreamBinder = new KStreamBinder(binderConfigurationProperties,
|
||||
kafkaTopicProvisioner, KafkaStreamsMessageConversionDelegate,
|
||||
KafkaStreamsBindingInformationCatalogue, keyValueSerdeResolver, kafkaStreamsRegistry);
|
||||
kStreamBinder.setKafkaStreamsExtendedBindingProperties(
|
||||
kafkaStreamsExtendedBindingProperties);
|
||||
return kStreamBinder;
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnBean(name = "outerContext")
|
||||
public static BeanFactoryPostProcessor outerContextBeanFactoryPostProcessor() {
|
||||
return beanFactory -> {
|
||||
|
||||
// It is safe to call getBean("outerContext") here, because this bean is
|
||||
// registered as first
|
||||
// and as independent from the parent context.
|
||||
ApplicationContext outerContext = (ApplicationContext) beanFactory
|
||||
.getBean("outerContext");
|
||||
beanFactory.registerSingleton(
|
||||
KafkaStreamsMessageConversionDelegate.class.getSimpleName(),
|
||||
outerContext.getBean(KafkaStreamsMessageConversionDelegate.class));
|
||||
beanFactory.registerSingleton(
|
||||
KafkaStreamsBindingInformationCatalogue.class.getSimpleName(),
|
||||
outerContext.getBean(KafkaStreamsBindingInformationCatalogue.class));
|
||||
beanFactory.registerSingleton(KeyValueSerdeResolver.class.getSimpleName(),
|
||||
outerContext.getBean(KeyValueSerdeResolver.class));
|
||||
beanFactory.registerSingleton(
|
||||
KafkaStreamsExtendedBindingProperties.class.getSimpleName(),
|
||||
outerContext.getBean(KafkaStreamsExtendedBindingProperties.class));
|
||||
beanFactory.registerSingleton(
|
||||
KafkaStreamsRegistry.class.getSimpleName(),
|
||||
outerContext.getBean(KafkaStreamsRegistry.class));
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
/*
|
||||
* Copyright 2017-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import org.aopalliance.intercept.MethodInterceptor;
|
||||
import org.aopalliance.intercept.MethodInvocation;
|
||||
import org.apache.kafka.streams.kstream.KStream;
|
||||
|
||||
import org.springframework.aop.framework.ProxyFactory;
|
||||
import org.springframework.cloud.stream.binder.ConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.ProducerProperties;
|
||||
import org.springframework.cloud.stream.binding.AbstractBindingTargetFactory;
|
||||
import org.springframework.cloud.stream.config.BindingProperties;
|
||||
import org.springframework.cloud.stream.config.BindingServiceProperties;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
/**
|
||||
* {@link org.springframework.cloud.stream.binding.BindingTargetFactory}
|
||||
* for{@link KStream}.
|
||||
*
|
||||
* The implementation creates proxies for both input and output binding. The actual target
|
||||
* will be created downstream through further binding process.
|
||||
*
|
||||
* @author Marius Bogoevici
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
class KStreamBoundElementFactory extends AbstractBindingTargetFactory<KStream> {
|
||||
|
||||
private final BindingServiceProperties bindingServiceProperties;
|
||||
|
||||
private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue;
|
||||
private final EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler;
|
||||
|
||||
KStreamBoundElementFactory(BindingServiceProperties bindingServiceProperties,
|
||||
KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue,
|
||||
EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler) {
|
||||
super(KStream.class);
|
||||
this.bindingServiceProperties = bindingServiceProperties;
|
||||
this.kafkaStreamsBindingInformationCatalogue = KafkaStreamsBindingInformationCatalogue;
|
||||
this.encodingDecodingBindAdviceHandler = encodingDecodingBindAdviceHandler;
|
||||
}
|
||||
|
||||
@Override
|
||||
public KStream createInput(String name) {
|
||||
BindingProperties bindingProperties = this.bindingServiceProperties.getBindingProperties(name);
|
||||
ConsumerProperties consumerProperties = bindingProperties.getConsumer();
|
||||
if (consumerProperties == null) {
|
||||
consumerProperties = this.bindingServiceProperties.getConsumerProperties(name);
|
||||
consumerProperties.setUseNativeDecoding(true);
|
||||
}
|
||||
else {
|
||||
if (!encodingDecodingBindAdviceHandler.isDecodingSettingProvided()) {
|
||||
consumerProperties.setUseNativeDecoding(true);
|
||||
}
|
||||
}
|
||||
// Always set multiplex to true in the kafka streams binder
|
||||
consumerProperties.setMultiplex(true);
|
||||
return createProxyForKStream(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
public KStream createOutput(final String name) {
|
||||
|
||||
BindingProperties bindingProperties = this.bindingServiceProperties.getBindingProperties(name);
|
||||
ProducerProperties producerProperties = bindingProperties.getProducer();
|
||||
if (producerProperties == null) {
|
||||
producerProperties = this.bindingServiceProperties.getProducerProperties(name);
|
||||
producerProperties.setUseNativeEncoding(true);
|
||||
}
|
||||
else {
|
||||
if (!encodingDecodingBindAdviceHandler.isEncodingSettingProvided()) {
|
||||
producerProperties.setUseNativeEncoding(true);
|
||||
}
|
||||
}
|
||||
return createProxyForKStream(name);
|
||||
}
|
||||
|
||||
private KStream createProxyForKStream(String name) {
|
||||
KStreamWrapperHandler wrapper = new KStreamWrapperHandler();
|
||||
ProxyFactory proxyFactory = new ProxyFactory(KStreamWrapper.class, KStream.class);
|
||||
proxyFactory.addAdvice(wrapper);
|
||||
|
||||
KStream proxy = (KStream) proxyFactory.getProxy();
|
||||
|
||||
// Add the binding properties to the catalogue for later retrieval during further
|
||||
// binding steps downstream.
|
||||
BindingProperties bindingProperties = this.bindingServiceProperties
|
||||
.getBindingProperties(name);
|
||||
this.kafkaStreamsBindingInformationCatalogue.registerBindingProperties(proxy,
|
||||
bindingProperties);
|
||||
this.kafkaStreamsBindingInformationCatalogue.addBindingNamePerTarget(proxy, name);
|
||||
return proxy;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrapper object for KStream proxy.
|
||||
*/
|
||||
public interface KStreamWrapper {
|
||||
|
||||
void wrap(KStream<Object, Object> delegate);
|
||||
|
||||
}
|
||||
|
||||
static class KStreamWrapperHandler
|
||||
implements KStreamWrapper, MethodInterceptor {
|
||||
|
||||
private KStream<Object, Object> delegate;
|
||||
|
||||
public void wrap(KStream<Object, Object> delegate) {
|
||||
Assert.notNull(delegate, "delegate cannot be null");
|
||||
if (this.delegate == null) {
|
||||
this.delegate = delegate;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object invoke(MethodInvocation methodInvocation) throws Throwable {
|
||||
if (methodInvocation.getMethod().getDeclaringClass().equals(KStream.class)) {
|
||||
Assert.notNull(this.delegate,
|
||||
"Trying to prepareConsumerBinding " + methodInvocation.getMethod()
|
||||
+ " but no delegate has been set.");
|
||||
return methodInvocation.getMethod().invoke(this.delegate,
|
||||
methodInvocation.getArguments());
|
||||
}
|
||||
else if (methodInvocation.getMethod().getDeclaringClass()
|
||||
.equals(KStreamWrapper.class)) {
|
||||
return methodInvocation.getMethod().invoke(this,
|
||||
methodInvocation.getArguments());
|
||||
}
|
||||
else {
|
||||
throw new IllegalStateException(
|
||||
"Only KStream method invocations are permitted");
|
||||
}
|
||||
}
|
||||
|
||||
KStream<Object, Object> getDelegate() {
|
||||
return delegate;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2021 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import org.apache.kafka.streams.KafkaStreams;
|
||||
import org.apache.kafka.streams.StreamsConfig;
|
||||
import org.apache.kafka.streams.kstream.KTable;
|
||||
|
||||
import org.springframework.cloud.stream.binder.AbstractBinder;
|
||||
import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider;
|
||||
import org.springframework.cloud.stream.binder.Binding;
|
||||
import org.springframework.cloud.stream.binder.DefaultBinding;
|
||||
import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsProducerProperties;
|
||||
import org.springframework.kafka.config.StreamsBuilderFactoryBean;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* {@link org.springframework.cloud.stream.binder.Binder} implementation for
|
||||
* {@link KTable}. This implemenation extends from the {@link AbstractBinder} directly.
|
||||
*
|
||||
* Provides only consumer binding for the bound KTable as output bindings are not allowed
|
||||
* on it.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
class KTableBinder extends
|
||||
AbstractBinder<KTable<Object, Object>, ExtendedConsumerProperties<KafkaStreamsConsumerProperties>, ExtendedProducerProperties<KafkaStreamsProducerProperties>>
|
||||
implements
|
||||
ExtendedPropertiesBinder<KTable<Object, Object>, KafkaStreamsConsumerProperties, KafkaStreamsProducerProperties> {
|
||||
|
||||
private final KafkaStreamsBinderConfigurationProperties binderConfigurationProperties;
|
||||
|
||||
private final KafkaTopicProvisioner kafkaTopicProvisioner;
|
||||
|
||||
private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue;
|
||||
|
||||
// @checkstyle:off
|
||||
private KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties = new KafkaStreamsExtendedBindingProperties();
|
||||
|
||||
// @checkstyle:on
|
||||
|
||||
private final KafkaStreamsRegistry kafkaStreamsRegistry;
|
||||
|
||||
KTableBinder(KafkaStreamsBinderConfigurationProperties binderConfigurationProperties,
|
||||
KafkaTopicProvisioner kafkaTopicProvisioner,
|
||||
KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue, KafkaStreamsRegistry kafkaStreamsRegistry) {
|
||||
this.binderConfigurationProperties = binderConfigurationProperties;
|
||||
this.kafkaTopicProvisioner = kafkaTopicProvisioner;
|
||||
this.kafkaStreamsBindingInformationCatalogue = KafkaStreamsBindingInformationCatalogue;
|
||||
this.kafkaStreamsRegistry = kafkaStreamsRegistry;
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
protected Binding<KTable<Object, Object>> doBindConsumer(String name, String group,
|
||||
KTable<Object, Object> inputTarget,
|
||||
// @checkstyle:off
|
||||
ExtendedConsumerProperties<KafkaStreamsConsumerProperties> properties) {
|
||||
// @checkstyle:on
|
||||
if (!StringUtils.hasText(group)) {
|
||||
group = properties.getExtension().getApplicationId();
|
||||
}
|
||||
|
||||
final RetryTemplate retryTemplate = buildRetryTemplate(properties);
|
||||
|
||||
final String bindingName = this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(inputTarget);
|
||||
final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsBindingInformationCatalogue
|
||||
.getStreamsBuilderFactoryBeanPerBinding().get(bindingName);
|
||||
|
||||
KafkaStreamsBinderUtils.prepareConsumerBinding(name, group,
|
||||
getApplicationContext(), this.kafkaTopicProvisioner,
|
||||
this.binderConfigurationProperties, properties, retryTemplate, getBeanFactory(),
|
||||
this.kafkaStreamsBindingInformationCatalogue.bindingNamePerTarget(inputTarget),
|
||||
this.kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean);
|
||||
|
||||
return new DefaultBinding<KTable<Object, Object>>(bindingName, group, inputTarget, streamsBuilderFactoryBean) {
|
||||
|
||||
@Override
|
||||
public boolean isInput() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void start() {
|
||||
if (!streamsBuilderFactoryBean.isRunning()) {
|
||||
super.start();
|
||||
KTableBinder.this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean);
|
||||
//If we cached the previous KafkaStreams object (from a binding stop on the actuator), remove it.
|
||||
//See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165
|
||||
final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG);
|
||||
if (kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().containsKey(applicationId)) {
|
||||
kafkaStreamsBindingInformationCatalogue.removePreviousKafkaStreamsForApplicationId(applicationId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void stop() {
|
||||
if (streamsBuilderFactoryBean.isRunning()) {
|
||||
final KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams();
|
||||
super.stop();
|
||||
KTableBinder.this.kafkaStreamsRegistry.unregisterKafkaStreams(kafkaStreams);
|
||||
KafkaStreamsBinderUtils.closeDlqProducerFactories(kafkaStreamsBindingInformationCatalogue, streamsBuilderFactoryBean);
|
||||
//Caching the stopped KafkaStreams for health indicator purposes on the underlying processor.
|
||||
//See this issue for more details: https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165
|
||||
KTableBinder.this.kafkaStreamsBindingInformationCatalogue.addPreviousKafkaStreamsForApplicationId(
|
||||
(String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG), kafkaStreams);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Binding<KTable<Object, Object>> doBindProducer(String name,
|
||||
KTable<Object, Object> outboundBindTarget,
|
||||
ExtendedProducerProperties<KafkaStreamsProducerProperties> properties) {
|
||||
throw new UnsupportedOperationException(
|
||||
"No producer level binding is allowed for KTable");
|
||||
}
|
||||
|
||||
@Override
|
||||
public KafkaStreamsConsumerProperties getExtendedConsumerProperties(
|
||||
String channelName) {
|
||||
return this.kafkaStreamsExtendedBindingProperties
|
||||
.getExtendedConsumerProperties(channelName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public KafkaStreamsProducerProperties getExtendedProducerProperties(
|
||||
String channelName) {
|
||||
return this.kafkaStreamsExtendedBindingProperties
|
||||
.getExtendedProducerProperties(channelName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDefaultsPrefix() {
|
||||
return this.kafkaStreamsExtendedBindingProperties.getDefaultsPrefix();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<? extends BinderSpecificPropertiesProvider> getExtendedPropertiesEntryClass() {
|
||||
return this.kafkaStreamsExtendedBindingProperties
|
||||
.getExtendedPropertiesEntryClass();
|
||||
}
|
||||
|
||||
public void setKafkaStreamsExtendedBindingProperties(
|
||||
KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties) {
|
||||
this.kafkaStreamsExtendedBindingProperties = kafkaStreamsExtendedBindingProperties;
|
||||
}
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
|
||||
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.AdminClientConfigCustomizer;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
/**
|
||||
* Configuration for KTable binder.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @author Chris Bono
|
||||
*/
|
||||
@SuppressWarnings("ALL")
|
||||
@Configuration
|
||||
@Import({ MultiBinderPropertiesConfiguration.class,
|
||||
KafkaStreamsBinderHealthIndicatorConfiguration.class,
|
||||
KafkaStreamsJaasConfiguration.class})
|
||||
@EnableConfigurationProperties(KafkaProperties.class)
|
||||
public class KTableBinderConfiguration {
|
||||
|
||||
@Bean
|
||||
public KafkaTopicProvisioner provisioningProvider(
|
||||
KafkaStreamsBinderConfigurationProperties binderConfigurationProperties,
|
||||
KafkaProperties kafkaProperties, ObjectProvider<AdminClientConfigCustomizer> adminClientConfigCustomizer) {
|
||||
return new KafkaTopicProvisioner(binderConfigurationProperties, kafkaProperties, adminClientConfigCustomizer.getIfUnique());
|
||||
}
|
||||
|
||||
@Bean
|
||||
public KTableBinder kTableBinder(
|
||||
KafkaStreamsBinderConfigurationProperties binderConfigurationProperties,
|
||||
KafkaTopicProvisioner kafkaTopicProvisioner,
|
||||
KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties,
|
||||
KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue,
|
||||
@Qualifier("streamConfigGlobalProperties") Map<String, Object> streamConfigGlobalProperties,
|
||||
KafkaStreamsRegistry kafkaStreamsRegistry) {
|
||||
KTableBinder kTableBinder = new KTableBinder(binderConfigurationProperties,
|
||||
kafkaTopicProvisioner, kafkaStreamsBindingInformationCatalogue, kafkaStreamsRegistry);
|
||||
kTableBinder.setKafkaStreamsExtendedBindingProperties(kafkaStreamsExtendedBindingProperties);
|
||||
return kTableBinder;
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnBean(name = "outerContext")
|
||||
public static BeanFactoryPostProcessor outerContextBeanFactoryPostProcessor() {
|
||||
return beanFactory -> {
|
||||
|
||||
// It is safe to call getBean("outerContext") here, because this bean is
|
||||
// registered as first
|
||||
// and as independent from the parent context.
|
||||
ApplicationContext outerContext = (ApplicationContext) beanFactory
|
||||
.getBean("outerContext");
|
||||
beanFactory.registerSingleton(
|
||||
KafkaStreamsExtendedBindingProperties.class.getSimpleName(),
|
||||
outerContext.getBean(KafkaStreamsExtendedBindingProperties.class));
|
||||
beanFactory.registerSingleton(
|
||||
KafkaStreamsBindingInformationCatalogue.class.getSimpleName(),
|
||||
outerContext.getBean(KafkaStreamsBindingInformationCatalogue.class));
|
||||
beanFactory.registerSingleton(
|
||||
KafkaStreamsRegistry.class.getSimpleName(),
|
||||
outerContext.getBean(KafkaStreamsRegistry.class));
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import org.aopalliance.intercept.MethodInterceptor;
|
||||
import org.aopalliance.intercept.MethodInvocation;
|
||||
import org.apache.kafka.streams.kstream.KTable;
|
||||
|
||||
import org.springframework.aop.framework.ProxyFactory;
|
||||
import org.springframework.cloud.stream.binder.ConsumerProperties;
|
||||
import org.springframework.cloud.stream.binding.AbstractBindingTargetFactory;
|
||||
import org.springframework.cloud.stream.config.BindingProperties;
|
||||
import org.springframework.cloud.stream.config.BindingServiceProperties;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
/**
|
||||
* {@link org.springframework.cloud.stream.binding.BindingTargetFactory} for
|
||||
* {@link KTable}
|
||||
*
|
||||
* Input bindings are only created as output bindings on KTable are not allowed.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
class KTableBoundElementFactory extends AbstractBindingTargetFactory<KTable> {
|
||||
|
||||
private final BindingServiceProperties bindingServiceProperties;
|
||||
private final EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler;
|
||||
private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue;
|
||||
|
||||
KTableBoundElementFactory(BindingServiceProperties bindingServiceProperties,
|
||||
EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler,
|
||||
KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue) {
|
||||
super(KTable.class);
|
||||
this.bindingServiceProperties = bindingServiceProperties;
|
||||
this.encodingDecodingBindAdviceHandler = encodingDecodingBindAdviceHandler;
|
||||
this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue;
|
||||
}
|
||||
|
||||
@Override
|
||||
public KTable createInput(String name) {
|
||||
BindingProperties bindingProperties = this.bindingServiceProperties.getBindingProperties(name);
|
||||
ConsumerProperties consumerProperties = bindingProperties.getConsumer();
|
||||
if (consumerProperties == null) {
|
||||
consumerProperties = this.bindingServiceProperties.getConsumerProperties(name);
|
||||
consumerProperties.setUseNativeDecoding(true);
|
||||
}
|
||||
else {
|
||||
if (!encodingDecodingBindAdviceHandler.isDecodingSettingProvided()) {
|
||||
consumerProperties.setUseNativeDecoding(true);
|
||||
}
|
||||
}
|
||||
// Always set multiplex to true in the kafka streams binder
|
||||
consumerProperties.setMultiplex(true);
|
||||
|
||||
KTableBoundElementFactory.KTableWrapperHandler wrapper = new KTableBoundElementFactory.KTableWrapperHandler();
|
||||
ProxyFactory proxyFactory = new ProxyFactory(
|
||||
KTableBoundElementFactory.KTableWrapper.class, KTable.class);
|
||||
proxyFactory.addAdvice(wrapper);
|
||||
|
||||
final KTable proxy = (KTable) proxyFactory.getProxy();
|
||||
this.kafkaStreamsBindingInformationCatalogue.addBindingNamePerTarget(proxy, name);
|
||||
return proxy;
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
public KTable createOutput(final String name) {
|
||||
throw new UnsupportedOperationException(
|
||||
"Outbound operations are not allowed on target type KTable");
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrapper for KTable proxy.
|
||||
*/
|
||||
public interface KTableWrapper {
|
||||
|
||||
void wrap(KTable<Object, Object> delegate);
|
||||
|
||||
}
|
||||
|
||||
private static class KTableWrapperHandler
|
||||
implements KTableBoundElementFactory.KTableWrapper, MethodInterceptor {
|
||||
|
||||
private KTable<Object, Object> delegate;
|
||||
|
||||
public void wrap(KTable<Object, Object> delegate) {
|
||||
Assert.notNull(delegate, "delegate cannot be null");
|
||||
if (this.delegate == null) {
|
||||
this.delegate = delegate;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object invoke(MethodInvocation methodInvocation) throws Throwable {
|
||||
if (methodInvocation.getMethod().getDeclaringClass().equals(KTable.class)) {
|
||||
Assert.notNull(this.delegate,
|
||||
"Trying to prepareConsumerBinding " + methodInvocation.getMethod()
|
||||
+ " but no delegate has been set.");
|
||||
return methodInvocation.getMethod().invoke(this.delegate,
|
||||
methodInvocation.getArguments());
|
||||
}
|
||||
else if (methodInvocation.getMethod().getDeclaringClass()
|
||||
.equals(KTableBoundElementFactory.KTableWrapper.class)) {
|
||||
return methodInvocation.getMethod().invoke(this,
|
||||
methodInvocation.getArguments());
|
||||
}
|
||||
else {
|
||||
throw new IllegalStateException(
|
||||
"Only KTable method invocations are permitted");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.Collections;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.env.EnvironmentPostProcessor;
|
||||
import org.springframework.core.env.ConfigurableEnvironment;
|
||||
import org.springframework.core.env.MapPropertySource;
|
||||
|
||||
/**
|
||||
* {@link EnvironmentPostProcessor} to ensure the {@link SendToDlqAndContinue sendToDlqAndContinue} BiFunction
|
||||
* is excluded in core Spring Cloud Function by adding it to the ineligible function definitions.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @author Chris Bono
|
||||
*/
|
||||
public class KafkaStreamsBinderEnvironmentPostProcessor implements EnvironmentPostProcessor {
|
||||
|
||||
/**
|
||||
* The bean name of the SendToDlqAndContinue function - must remain in sync w/
|
||||
* {@link KafkaStreamsBinderSupportAutoConfiguration#sendToDlqAndContinue()}.
|
||||
*/
|
||||
private static final String SEND_TO_DLQ_AND_CONTINUE_BEAN_NAME = "sendToDlqAndContinue";
|
||||
|
||||
@Override
|
||||
public void postProcessEnvironment(ConfigurableEnvironment environment, SpringApplication application) {
|
||||
String ineligibleDefinitionsPropertyKey = "spring.cloud.function.ineligible-definitions";
|
||||
String ineligibleDefinitions = SEND_TO_DLQ_AND_CONTINUE_BEAN_NAME;
|
||||
if (environment.getProperty(ineligibleDefinitionsPropertyKey) != null) {
|
||||
ineligibleDefinitions += ("," + environment.getProperty(ineligibleDefinitionsPropertyKey));
|
||||
}
|
||||
environment.getPropertySources().addFirst(new MapPropertySource(
|
||||
"kafkaStreamsBinderIneligibleDefinitions",
|
||||
Collections.singletonMap(ineligibleDefinitionsPropertyKey, ineligibleDefinitions)));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,232 +0,0 @@
|
||||
/*
|
||||
* Copyright 2019-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.time.Duration;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.kafka.clients.admin.AdminClient;
|
||||
import org.apache.kafka.clients.admin.ListTopicsResult;
|
||||
import org.apache.kafka.streams.KafkaStreams;
|
||||
import org.apache.kafka.streams.StreamsConfig;
|
||||
import org.apache.kafka.streams.TaskMetadata;
|
||||
import org.apache.kafka.streams.ThreadMetadata;
|
||||
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.boot.actuate.health.AbstractHealthIndicator;
|
||||
import org.springframework.boot.actuate.health.Health;
|
||||
import org.springframework.boot.actuate.health.Status;
|
||||
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.kafka.config.StreamsBuilderFactoryBean;
|
||||
|
||||
/**
|
||||
* Health indicator for Kafka Streams.
|
||||
*
|
||||
* @author Arnaud Jardiné
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
public class KafkaStreamsBinderHealthIndicator extends AbstractHealthIndicator implements DisposableBean {
|
||||
|
||||
/**
|
||||
* Static initialization for detecting whether the application is using Kafka client 2.5 vs lower versions.
|
||||
*/
|
||||
private static ClassLoader CLASS_LOADER = KafkaStreamsBinderHealthIndicator.class.getClassLoader();
|
||||
private static boolean isKafkaStreams25 = true;
|
||||
private static Method methodForIsRunning;
|
||||
|
||||
static {
|
||||
try {
|
||||
Class<?> KAFKA_STREAMS_STATE_CLASS = CLASS_LOADER.loadClass("org.apache.kafka.streams.KafkaStreams$State");
|
||||
|
||||
Method[] declaredMethods = KAFKA_STREAMS_STATE_CLASS.getDeclaredMethods();
|
||||
for (Method m : declaredMethods) {
|
||||
if (m.getName().equals("isRunning")) {
|
||||
isKafkaStreams25 = false;
|
||||
methodForIsRunning = m;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (ClassNotFoundException e) {
|
||||
throw new IllegalStateException("KafkaStreams$State class not found", e);
|
||||
}
|
||||
}
|
||||
|
||||
private final KafkaStreamsRegistry kafkaStreamsRegistry;
|
||||
|
||||
private final KafkaStreamsBinderConfigurationProperties configurationProperties;
|
||||
|
||||
private final Map<String, Object> adminClientProperties;
|
||||
|
||||
private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue;
|
||||
|
||||
private AdminClient adminClient;
|
||||
|
||||
private final Lock lock = new ReentrantLock();
|
||||
|
||||
KafkaStreamsBinderHealthIndicator(KafkaStreamsRegistry kafkaStreamsRegistry,
|
||||
KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties,
|
||||
KafkaProperties kafkaProperties,
|
||||
KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue) {
|
||||
super("Kafka-streams health check failed");
|
||||
kafkaProperties.buildAdminProperties();
|
||||
this.configurationProperties = kafkaStreamsBinderConfigurationProperties;
|
||||
this.adminClientProperties = kafkaProperties.buildAdminProperties();
|
||||
KafkaTopicProvisioner.normalalizeBootPropsWithBinder(this.adminClientProperties, kafkaProperties,
|
||||
kafkaStreamsBinderConfigurationProperties);
|
||||
this.kafkaStreamsRegistry = kafkaStreamsRegistry;
|
||||
this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doHealthCheck(Health.Builder builder) throws Exception {
|
||||
try {
|
||||
this.lock.lock();
|
||||
if (this.adminClient == null) {
|
||||
this.adminClient = AdminClient.create(this.adminClientProperties);
|
||||
}
|
||||
|
||||
final ListTopicsResult listTopicsResult = this.adminClient.listTopics();
|
||||
listTopicsResult.listings().get(this.configurationProperties.getHealthTimeout(), TimeUnit.SECONDS);
|
||||
|
||||
if (this.kafkaStreamsBindingInformationCatalogue.getStreamsBuilderFactoryBeans().isEmpty()) {
|
||||
builder.withDetail("No Kafka Streams bindings have been established", "Kafka Streams binder did not detect any processors");
|
||||
builder.status(Status.UNKNOWN);
|
||||
}
|
||||
else {
|
||||
boolean up = true;
|
||||
final Set<KafkaStreams> kafkaStreams = kafkaStreamsRegistry.getKafkaStreams();
|
||||
Set<KafkaStreams> allKafkaStreams = new HashSet<>(kafkaStreams);
|
||||
if (this.configurationProperties.isIncludeStoppedProcessorsForHealthCheck()) {
|
||||
allKafkaStreams.addAll(kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams().values());
|
||||
}
|
||||
for (KafkaStreams kStream : allKafkaStreams) {
|
||||
if (isKafkaStreams25) {
|
||||
up &= kStream.state().isRunningOrRebalancing();
|
||||
}
|
||||
else {
|
||||
// if Kafka client version is lower than 2.5, then call the method reflectively.
|
||||
final boolean isRunningInvokedResult = (boolean) methodForIsRunning.invoke(kStream.state());
|
||||
up &= isRunningInvokedResult;
|
||||
}
|
||||
builder.withDetails(buildDetails(kStream));
|
||||
}
|
||||
builder.status(up ? Status.UP : Status.DOWN);
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
builder.withDetail("No topic information available", "Kafka broker is not reachable");
|
||||
builder.status(Status.DOWN);
|
||||
builder.withException(e);
|
||||
}
|
||||
finally {
|
||||
this.lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> buildDetails(KafkaStreams kafkaStreams) throws Exception {
|
||||
final Map<String, Object> details = new HashMap<>();
|
||||
final Map<String, Object> perAppdIdDetails = new HashMap<>();
|
||||
|
||||
boolean isRunningResult;
|
||||
if (isKafkaStreams25) {
|
||||
isRunningResult = kafkaStreams.state().isRunningOrRebalancing();
|
||||
}
|
||||
else {
|
||||
// if Kafka client version is lower than 2.5, then call the method reflectively.
|
||||
isRunningResult = (boolean) methodForIsRunning.invoke(kafkaStreams.state());
|
||||
}
|
||||
|
||||
if (isRunningResult) {
|
||||
final Set<ThreadMetadata> threadMetadata = kafkaStreams.metadataForLocalThreads();
|
||||
final Map<String, Object> threadDetails = new HashMap<>();
|
||||
for (ThreadMetadata metadata : threadMetadata) {
|
||||
final Map<String, Object> threadDetail = new HashMap<>();
|
||||
threadDetail.put("threadName", metadata.threadName());
|
||||
threadDetail.put("threadState", metadata.threadState());
|
||||
threadDetail.put("adminClientId", metadata.adminClientId());
|
||||
threadDetail.put("consumerClientId", metadata.consumerClientId());
|
||||
threadDetail.put("restoreConsumerClientId", metadata.restoreConsumerClientId());
|
||||
threadDetail.put("producerClientIds", metadata.producerClientIds());
|
||||
threadDetail.put("activeTasks", taskDetails(metadata.activeTasks()));
|
||||
threadDetail.put("standbyTasks", taskDetails(metadata.standbyTasks()));
|
||||
threadDetails.put(metadata.threadName(), threadDetail);
|
||||
}
|
||||
perAppdIdDetails.put("threadDetails", threadDetails);
|
||||
final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsRegistry.streamBuilderFactoryBean(kafkaStreams);
|
||||
final String applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG);
|
||||
details.put(applicationId, perAppdIdDetails);
|
||||
}
|
||||
else {
|
||||
final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsRegistry.streamBuilderFactoryBean(kafkaStreams);
|
||||
String applicationId = null;
|
||||
if (streamsBuilderFactoryBean != null) {
|
||||
applicationId = (String) streamsBuilderFactoryBean.getStreamsConfiguration().get(StreamsConfig.APPLICATION_ID_CONFIG);
|
||||
}
|
||||
else {
|
||||
final Map<String, KafkaStreams> stoppedKafkaStreamsPerBinding = kafkaStreamsBindingInformationCatalogue.getStoppedKafkaStreams();
|
||||
for (String appId : stoppedKafkaStreamsPerBinding.keySet()) {
|
||||
if (stoppedKafkaStreamsPerBinding.get(appId).equals(kafkaStreams)) {
|
||||
applicationId = appId;
|
||||
}
|
||||
}
|
||||
}
|
||||
details.put(applicationId, String.format("The processor with application.id %s is down. Current state: %s", applicationId, kafkaStreams.state()));
|
||||
}
|
||||
return details;
|
||||
}
|
||||
|
||||
private static Map<String, Object> taskDetails(Set<TaskMetadata> taskMetadata) {
|
||||
final Map<String, Object> details = new HashMap<>();
|
||||
for (TaskMetadata metadata : taskMetadata) {
|
||||
details.put("taskId", metadata.taskId());
|
||||
if (details.containsKey("partitions")) {
|
||||
@SuppressWarnings("unchecked")
|
||||
List<String> partitionsInfo = (List<String>) details.get("partitions");
|
||||
partitionsInfo.addAll(addPartitionsInfo(metadata));
|
||||
}
|
||||
else {
|
||||
details.put("partitions",
|
||||
addPartitionsInfo(metadata));
|
||||
}
|
||||
}
|
||||
return details;
|
||||
}
|
||||
|
||||
private static List<String> addPartitionsInfo(TaskMetadata metadata) {
|
||||
return metadata.topicPartitions().stream().map(
|
||||
p -> "partition=" + p.partition() + ", topic=" + p.topic())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
if (adminClient != null) {
|
||||
adminClient.close(Duration.ofSeconds(0));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
/*
|
||||
* Copyright 2019-2021 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.actuate.autoconfigure.health.ConditionalOnEnabledHealthIndicator;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
|
||||
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* Configuration class for Kafka-streams binder health indicator beans.
|
||||
*
|
||||
* @author Arnaud Jardiné
|
||||
*/
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@ConditionalOnClass(name = "org.springframework.boot.actuate.health.HealthIndicator")
|
||||
@ConditionalOnEnabledHealthIndicator("binders")
|
||||
public class KafkaStreamsBinderHealthIndicatorConfiguration {
|
||||
|
||||
@Bean
|
||||
public KafkaStreamsBinderHealthIndicator kafkaStreamsBinderHealthIndicator(
|
||||
ObjectProvider<KafkaStreamsRegistry> kafkaStreamsRegistry,
|
||||
@Qualifier("binderConfigurationProperties")KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties,
|
||||
KafkaProperties kafkaProperties, KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue) {
|
||||
if (kafkaStreamsRegistry.getIfUnique() != null) {
|
||||
return new KafkaStreamsBinderHealthIndicator(kafkaStreamsRegistry.getIfUnique(), kafkaStreamsBinderConfigurationProperties,
|
||||
kafkaProperties, kafkaStreamsBindingInformationCatalogue);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,240 +0,0 @@
|
||||
/*
|
||||
* Copyright 2019-2019 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.function.ToDoubleFunction;
|
||||
|
||||
import io.micrometer.core.instrument.FunctionCounter;
|
||||
import io.micrometer.core.instrument.Gauge;
|
||||
import io.micrometer.core.instrument.Meter;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import io.micrometer.core.instrument.Tag;
|
||||
import io.micrometer.core.instrument.binder.MeterBinder;
|
||||
import org.apache.kafka.common.Metric;
|
||||
import org.apache.kafka.common.MetricName;
|
||||
import org.apache.kafka.streams.KafkaStreams;
|
||||
|
||||
import org.springframework.kafka.config.StreamsBuilderFactoryBean;
|
||||
|
||||
/**
|
||||
* Kafka Streams binder metrics implementation that exports the metrics available
|
||||
* through {@link KafkaStreams#metrics()} into a micrometer {@link io.micrometer.core.instrument.MeterRegistry}.
|
||||
*
|
||||
* Boot 2.2 users need to rely on this class for the metrics instead of direct support from Micrometer.
|
||||
* Micrometer added Kafka Streams metrics support in 1.4.0 which Boot 2.3 includes.
|
||||
* Therefore, the users who are on Boot 2.2, need to rely on these metrics.
|
||||
* For users who are on 2.3 Boot, this class won't be activated (See the configuration for the various
|
||||
* conditionals used).
|
||||
*
|
||||
* For the most part, this class is a copy of the Micrometer Kafka Streams support that was added in version 1.4.0.
|
||||
* We will keep this class, as long as we support Boot 2.2.x.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @since 3.0.0
|
||||
*/
|
||||
public class KafkaStreamsBinderMetrics {
|
||||
|
||||
static final String DEFAULT_VALUE = "unknown";
|
||||
|
||||
static final String CLIENT_ID_TAG_NAME = "client-id";
|
||||
|
||||
static final String METRIC_GROUP_APP_INFO = "app-info";
|
||||
|
||||
static final String VERSION_METRIC_NAME = "version";
|
||||
|
||||
static final String START_TIME_METRIC_NAME = "start-time-ms";
|
||||
|
||||
static final String KAFKA_VERSION_TAG_NAME = "kafka-version";
|
||||
|
||||
static final String METRIC_NAME_PREFIX = "kafka.";
|
||||
|
||||
static final String METRIC_GROUP_METRICS_COUNT = "kafka-metrics-count";
|
||||
|
||||
private String kafkaVersion = DEFAULT_VALUE;
|
||||
|
||||
private String clientId = DEFAULT_VALUE;
|
||||
|
||||
private final MeterRegistry meterRegistry;
|
||||
|
||||
private MeterBinder meterBinder;
|
||||
|
||||
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
|
||||
|
||||
private volatile Set<MetricName> currentMeters = new HashSet<>();
|
||||
|
||||
public KafkaStreamsBinderMetrics(MeterRegistry meterRegistry) {
|
||||
this.meterRegistry = meterRegistry;
|
||||
}
|
||||
|
||||
public void bindTo(Set<StreamsBuilderFactoryBean> streamsBuilderFactoryBeans) {
|
||||
if (this.meterBinder == null) {
|
||||
this.meterBinder = registry -> {
|
||||
if (streamsBuilderFactoryBeans != null) {
|
||||
for (StreamsBuilderFactoryBean streamsBuilderFactoryBean : streamsBuilderFactoryBeans) {
|
||||
if (streamsBuilderFactoryBean.isRunning()) {
|
||||
KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams();
|
||||
final Map<MetricName, ? extends Metric> metrics = kafkaStreams.metrics();
|
||||
|
||||
prepareToBindMetrics(registry, metrics);
|
||||
checkAndBindMetrics(registry, metrics);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
this.meterBinder.bindTo(this.meterRegistry);
|
||||
}
|
||||
|
||||
public void addMetrics(Set<StreamsBuilderFactoryBean> streamsBuilderFactoryBeans) {
|
||||
synchronized (KafkaStreamsBinderMetrics.this) {
|
||||
this.bindTo(streamsBuilderFactoryBeans);
|
||||
}
|
||||
}
|
||||
|
||||
void prepareToBindMetrics(MeterRegistry registry, Map<MetricName, ? extends Metric> metrics) {
|
||||
Metric startTime = null;
|
||||
for (Map.Entry<MetricName, ? extends Metric> entry : metrics.entrySet()) {
|
||||
MetricName name = entry.getKey();
|
||||
if (clientId.equals(DEFAULT_VALUE) && name.tags().get(CLIENT_ID_TAG_NAME) != null) {
|
||||
clientId = name.tags().get(CLIENT_ID_TAG_NAME);
|
||||
}
|
||||
if (METRIC_GROUP_APP_INFO.equals(name.group())) {
|
||||
if (VERSION_METRIC_NAME.equals(name.name())) {
|
||||
kafkaVersion = (String) entry.getValue().metricValue();
|
||||
}
|
||||
else if (START_TIME_METRIC_NAME.equals(name.name())) {
|
||||
startTime = entry.getValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
if (startTime != null) {
|
||||
bindMeter(registry, startTime, meterName(startTime), meterTags(startTime));
|
||||
}
|
||||
}
|
||||
|
||||
private void bindMeter(MeterRegistry registry, Metric metric, String name, Iterable<Tag> tags) {
|
||||
if (name.endsWith("total") || name.endsWith("count")) {
|
||||
registerCounter(registry, metric, name, tags);
|
||||
}
|
||||
else {
|
||||
registerGauge(registry, metric, name, tags);
|
||||
}
|
||||
}
|
||||
|
||||
private void registerCounter(MeterRegistry registry, Metric metric, String name, Iterable<Tag> tags) {
|
||||
FunctionCounter.builder(name, metric, toMetricValue())
|
||||
.tags(tags)
|
||||
.description(metric.metricName().description())
|
||||
.register(registry);
|
||||
}
|
||||
|
||||
private ToDoubleFunction<Metric> toMetricValue() {
|
||||
return metric -> ((Number) metric.metricValue()).doubleValue();
|
||||
}
|
||||
|
||||
private void registerGauge(MeterRegistry registry, Metric metric, String name, Iterable<Tag> tags) {
|
||||
Gauge.builder(name, metric, toMetricValue())
|
||||
.tags(tags)
|
||||
.description(metric.metricName().description())
|
||||
.register(registry);
|
||||
}
|
||||
|
||||
private List<Tag> meterTags(Metric metric) {
|
||||
return meterTags(metric, false);
|
||||
}
|
||||
|
||||
private String meterName(Metric metric) {
|
||||
String name = METRIC_NAME_PREFIX + metric.metricName().group() + "." + metric.metricName().name();
|
||||
return name.replaceAll("-metrics", "").replaceAll("-", ".");
|
||||
}
|
||||
|
||||
private List<Tag> meterTags(Metric metric, boolean includeCommonTags) {
|
||||
List<Tag> tags = new ArrayList<>();
|
||||
metric.metricName().tags().forEach((key, value) -> tags.add(Tag.of(key, value)));
|
||||
tags.add(Tag.of(KAFKA_VERSION_TAG_NAME, kafkaVersion));
|
||||
return tags;
|
||||
}
|
||||
|
||||
private boolean differentClient(List<Tag> tags) {
|
||||
for (Tag tag : tags) {
|
||||
if (tag.getKey().equals(CLIENT_ID_TAG_NAME)) {
|
||||
if (!clientId.equals(tag.getValue())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void checkAndBindMetrics(MeterRegistry registry, Map<MetricName, ? extends Metric> metrics) {
|
||||
if (!currentMeters.equals(metrics.keySet())) {
|
||||
currentMeters = new HashSet<>(metrics.keySet());
|
||||
metrics.forEach((name, metric) -> {
|
||||
//Filter out non-numeric values
|
||||
if (!(metric.metricValue() instanceof Number)) {
|
||||
return;
|
||||
}
|
||||
|
||||
//Filter out metrics from groups that include metadata
|
||||
if (METRIC_GROUP_APP_INFO.equals(name.group())) {
|
||||
return;
|
||||
}
|
||||
if (METRIC_GROUP_METRICS_COUNT.equals(name.group())) {
|
||||
return;
|
||||
}
|
||||
String meterName = meterName(metric);
|
||||
List<Tag> meterTagsWithCommonTags = meterTags(metric, true);
|
||||
//Kafka has metrics with lower number of tags (e.g. with/without topic or partition tag)
|
||||
//Remove meters with lower number of tags
|
||||
boolean hasLessTags = false;
|
||||
for (Meter other : registry.find(meterName).meters()) {
|
||||
List<Tag> tags = other.getId().getTags();
|
||||
// Only consider meters from the same client before filtering
|
||||
if (differentClient(tags)) {
|
||||
break;
|
||||
}
|
||||
if (tags.size() < meterTagsWithCommonTags.size()) {
|
||||
registry.remove(other);
|
||||
}
|
||||
// Check if already exists
|
||||
else if (tags.size() == meterTagsWithCommonTags.size()) {
|
||||
if (tags.equals(meterTagsWithCommonTags)) {
|
||||
return;
|
||||
}
|
||||
else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
hasLessTags = true;
|
||||
}
|
||||
}
|
||||
if (hasLessTags) {
|
||||
return;
|
||||
}
|
||||
bindMeter(registry, metric, meterName, meterTags(metric));
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,453 +0,0 @@
|
||||
/*
|
||||
* Copyright 2017-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import org.apache.kafka.common.serialization.Serdes;
|
||||
import org.apache.kafka.streams.StreamsConfig;
|
||||
import org.apache.kafka.streams.errors.LogAndContinueExceptionHandler;
|
||||
import org.apache.kafka.streams.errors.LogAndFailExceptionHandler;
|
||||
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingClass;
|
||||
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.bind.BindResult;
|
||||
import org.springframework.boot.context.properties.bind.Bindable;
|
||||
import org.springframework.boot.context.properties.bind.Binder;
|
||||
import org.springframework.boot.context.properties.bind.PropertySourcesPlaceholdersResolver;
|
||||
import org.springframework.boot.context.properties.source.ConfigurationPropertySources;
|
||||
import org.springframework.cloud.stream.binder.BinderConfiguration;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.function.FunctionDetectorCondition;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties;
|
||||
import org.springframework.cloud.stream.binding.BindingService;
|
||||
import org.springframework.cloud.stream.config.BinderProperties;
|
||||
import org.springframework.cloud.stream.config.BindingServiceConfiguration;
|
||||
import org.springframework.cloud.stream.config.BindingServiceProperties;
|
||||
import org.springframework.cloud.stream.function.StreamFunctionProperties;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Conditional;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.core.env.ConfigurableEnvironment;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.core.env.MapPropertySource;
|
||||
import org.springframework.integration.context.IntegrationContextUtils;
|
||||
import org.springframework.integration.support.utils.IntegrationUtils;
|
||||
import org.springframework.kafka.config.KafkaStreamsConfiguration;
|
||||
import org.springframework.kafka.config.StreamsBuilderFactoryBeanConfigurer;
|
||||
import org.springframework.kafka.core.CleanupConfig;
|
||||
import org.springframework.kafka.streams.KafkaStreamsMicrometerListener;
|
||||
import org.springframework.kafka.streams.RecoveringDeserializationExceptionHandler;
|
||||
import org.springframework.lang.Nullable;
|
||||
import org.springframework.messaging.converter.CompositeMessageConverter;
|
||||
import org.springframework.util.ObjectUtils;
|
||||
import org.springframework.util.ReflectionUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* Kafka Streams binder configuration.
|
||||
*
|
||||
* @author Marius Bogoevici
|
||||
* @author Soby Chacko
|
||||
* @author Gary Russell
|
||||
* @author Byungjun You
|
||||
*/
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@EnableConfigurationProperties({ KafkaProperties.class, KafkaStreamsExtendedBindingProperties.class })
|
||||
@ConditionalOnBean(BindingService.class)
|
||||
@AutoConfigureAfter(BindingServiceConfiguration.class)
|
||||
public class KafkaStreamsBinderSupportAutoConfiguration {
|
||||
|
||||
private static final String KSTREAM_BINDER_TYPE = "kstream";
|
||||
|
||||
private static final String KTABLE_BINDER_TYPE = "ktable";
|
||||
|
||||
private static final String GLOBALKTABLE_BINDER_TYPE = "globalktable";
|
||||
|
||||
private static final String CONSUMER_PROPERTIES_PREFIX = "consumer.";
|
||||
private static final String PRODUCER_PROPERTIES_PREFIX = "producer.";
|
||||
|
||||
@Bean
|
||||
@ConfigurationProperties(prefix = "spring.cloud.stream.kafka.streams.binder")
|
||||
public KafkaStreamsBinderConfigurationProperties binderConfigurationProperties(
|
||||
KafkaProperties kafkaProperties, ConfigurableEnvironment environment,
|
||||
BindingServiceProperties properties, ConfigurableApplicationContext context) throws Exception {
|
||||
final Map<String, BinderConfiguration> binderConfigurations = getBinderConfigurations(
|
||||
properties);
|
||||
for (Map.Entry<String, BinderConfiguration> entry : binderConfigurations
|
||||
.entrySet()) {
|
||||
final BinderConfiguration binderConfiguration = entry.getValue();
|
||||
final String binderType = binderConfiguration.getBinderType();
|
||||
if (binderType != null && (binderType.equals(KSTREAM_BINDER_TYPE)
|
||||
|| binderType.equals(KTABLE_BINDER_TYPE)
|
||||
|| binderType.equals(GLOBALKTABLE_BINDER_TYPE))) {
|
||||
Map<String, Object> binderProperties = new HashMap<>();
|
||||
this.flatten(null, binderConfiguration.getProperties(), binderProperties);
|
||||
environment.getPropertySources().addFirst(
|
||||
new MapPropertySource(entry.getKey() + "-kafkaStreamsBinderEnv", binderProperties));
|
||||
|
||||
Binder binder = new Binder(ConfigurationPropertySources.get(environment),
|
||||
new PropertySourcesPlaceholdersResolver(environment),
|
||||
IntegrationUtils.getConversionService(context.getBeanFactory()), null);
|
||||
final Constructor<KafkaStreamsBinderConfigurationProperties> kafkaStreamsBinderConfigurationPropertiesConstructor =
|
||||
ReflectionUtils.accessibleConstructor(KafkaStreamsBinderConfigurationProperties.class, KafkaProperties.class);
|
||||
final KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties =
|
||||
BeanUtils.instantiateClass(kafkaStreamsBinderConfigurationPropertiesConstructor, kafkaProperties);
|
||||
final BindResult<KafkaStreamsBinderConfigurationProperties> bind = binder.bind("spring.cloud.stream.kafka.streams.binder", Bindable.ofInstance(kafkaStreamsBinderConfigurationProperties));
|
||||
context.getBeanFactory().registerSingleton(
|
||||
entry.getKey() + "-KafkaStreamsBinderConfigurationProperties",
|
||||
bind.get());
|
||||
}
|
||||
}
|
||||
return new KafkaStreamsBinderConfigurationProperties(kafkaProperties);
|
||||
}
|
||||
|
||||
// TODO: Lifted from core - good candidate for exposing as a utility method in core.
|
||||
private static Map<String, BinderConfiguration> getBinderConfigurations(
|
||||
BindingServiceProperties properties) {
|
||||
|
||||
Map<String, BinderConfiguration> binderConfigurations = new HashMap<>();
|
||||
Map<String, BinderProperties> declaredBinders = properties.getBinders();
|
||||
|
||||
for (Map.Entry<String, BinderProperties> binderEntry : declaredBinders
|
||||
.entrySet()) {
|
||||
BinderProperties binderProperties = binderEntry.getValue();
|
||||
binderConfigurations.put(binderEntry.getKey(),
|
||||
new BinderConfiguration(binderProperties.getType(),
|
||||
binderProperties.getEnvironment(),
|
||||
binderProperties.isInheritEnvironment(),
|
||||
binderProperties.isDefaultCandidate()));
|
||||
}
|
||||
return binderConfigurations;
|
||||
}
|
||||
|
||||
// TODO: Lifted from core - good candidate for exposing as a utility method in core.
|
||||
@SuppressWarnings("unchecked")
|
||||
private void flatten(String propertyName, Object value,
|
||||
Map<String, Object> flattenedProperties) {
|
||||
if (value instanceof Map valueAsMap) {
|
||||
valueAsMap.forEach((k, v) -> flatten(
|
||||
(propertyName != null ? propertyName + "." : "") + k, v,
|
||||
flattenedProperties));
|
||||
}
|
||||
else {
|
||||
flattenedProperties.put(propertyName, value.toString());
|
||||
}
|
||||
}
|
||||
|
||||
@Bean
|
||||
public KafkaStreamsConfiguration kafkaStreamsConfiguration(
|
||||
@Qualifier("binderConfigurationProperties") KafkaStreamsBinderConfigurationProperties properties,
|
||||
Environment environment) {
|
||||
KafkaProperties kafkaProperties = properties.getKafkaProperties();
|
||||
Map<String, Object> streamsProperties = kafkaProperties.buildStreamsProperties();
|
||||
if (kafkaProperties.getStreams().getApplicationId() == null) {
|
||||
String applicationName = environment.getProperty("spring.application.name");
|
||||
if (applicationName != null) {
|
||||
streamsProperties.put(StreamsConfig.APPLICATION_ID_CONFIG,
|
||||
applicationName);
|
||||
}
|
||||
}
|
||||
return new KafkaStreamsConfiguration(streamsProperties);
|
||||
}
|
||||
|
||||
@Bean("streamConfigGlobalProperties")
|
||||
public Map<String, Object> streamConfigGlobalProperties(
|
||||
@Qualifier("binderConfigurationProperties") KafkaStreamsBinderConfigurationProperties configProperties,
|
||||
KafkaStreamsConfiguration kafkaStreamsConfiguration, ConfigurableEnvironment environment,
|
||||
SendToDlqAndContinue sendToDlqAndContinue) {
|
||||
|
||||
Properties properties = kafkaStreamsConfiguration.asProperties();
|
||||
|
||||
String kafkaConnectionString = configProperties.getKafkaConnectionString();
|
||||
|
||||
if (kafkaConnectionString != null && kafkaConnectionString.equals("localhost:9092")) {
|
||||
//Making sure that the application indeed set a property.
|
||||
String kafkaStreamsBinderBroker = environment.getProperty("spring.cloud.stream.kafka.streams.binder.brokers");
|
||||
|
||||
if (!StringUtils.hasText(kafkaStreamsBinderBroker)) {
|
||||
//Kafka Streams binder specific property for brokers is not set by the application.
|
||||
//See if there is one configured at the kafka binder level.
|
||||
String kafkaBinderBroker = environment.getProperty("spring.cloud.stream.kafka.binder.brokers");
|
||||
if (StringUtils.hasText(kafkaBinderBroker)) {
|
||||
kafkaConnectionString = kafkaBinderBroker;
|
||||
configProperties.setBrokers(kafkaConnectionString);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ObjectUtils.isEmpty(properties.get(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG))) {
|
||||
properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,
|
||||
kafkaConnectionString);
|
||||
}
|
||||
else {
|
||||
Object bootstrapServerConfig = properties
|
||||
.get(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG);
|
||||
if (bootstrapServerConfig instanceof String bootStrapServers) {
|
||||
if (bootStrapServers.equals("localhost:9092")) {
|
||||
properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,
|
||||
kafkaConnectionString);
|
||||
}
|
||||
}
|
||||
else if (bootstrapServerConfig instanceof List bootStrapCollection) {
|
||||
if (bootStrapCollection.size() == 1 && bootStrapCollection.get(0).equals("localhost:9092")) {
|
||||
properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,
|
||||
kafkaConnectionString);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
String binderProvidedApplicationId = configProperties.getApplicationId();
|
||||
if (StringUtils.hasText(binderProvidedApplicationId)) {
|
||||
properties.put(StreamsConfig.APPLICATION_ID_CONFIG,
|
||||
binderProvidedApplicationId);
|
||||
}
|
||||
|
||||
properties.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG,
|
||||
Serdes.ByteArraySerde.class.getName());
|
||||
properties.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG,
|
||||
Serdes.ByteArraySerde.class.getName());
|
||||
|
||||
if (configProperties
|
||||
.getDeserializationExceptionHandler() == DeserializationExceptionHandler.logAndContinue) {
|
||||
properties.put(
|
||||
StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG,
|
||||
LogAndContinueExceptionHandler.class);
|
||||
}
|
||||
else if (configProperties
|
||||
.getDeserializationExceptionHandler() == DeserializationExceptionHandler.logAndFail) {
|
||||
properties.put(
|
||||
StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG,
|
||||
LogAndFailExceptionHandler.class);
|
||||
}
|
||||
else if (configProperties
|
||||
.getDeserializationExceptionHandler() == DeserializationExceptionHandler.sendToDlq) {
|
||||
properties.put(
|
||||
StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG,
|
||||
RecoveringDeserializationExceptionHandler.class);
|
||||
properties.put(RecoveringDeserializationExceptionHandler.KSTREAM_DESERIALIZATION_RECOVERER, sendToDlqAndContinue);
|
||||
}
|
||||
|
||||
if (!ObjectUtils.isEmpty(configProperties.getConfiguration())) {
|
||||
properties.putAll(configProperties.getConfiguration());
|
||||
}
|
||||
|
||||
Map<String, Object> mergedConsumerConfig = new HashMap<>(configProperties.mergedConsumerConfiguration());
|
||||
//Adding consumer. prefix if they are missing (in order to differentiate them from other property categories such as stream, producer etc.)
|
||||
addPrefix(properties, mergedConsumerConfig, CONSUMER_PROPERTIES_PREFIX);
|
||||
|
||||
Map<String, Object> mergedProducerConfig = new HashMap<>(configProperties.mergedProducerConfiguration());
|
||||
//Adding producer. prefix if they are missing (in order to differentiate them from other property categories such as stream, consumer etc.)
|
||||
addPrefix(properties, mergedProducerConfig, PRODUCER_PROPERTIES_PREFIX);
|
||||
|
||||
if (!properties.containsKey(StreamsConfig.REPLICATION_FACTOR_CONFIG)) {
|
||||
properties.put(StreamsConfig.REPLICATION_FACTOR_CONFIG,
|
||||
(int) configProperties.getReplicationFactor());
|
||||
}
|
||||
return properties.entrySet().stream().collect(
|
||||
Collectors.toMap((e) -> String.valueOf(e.getKey()), Map.Entry::getValue));
|
||||
}
|
||||
|
||||
private void addPrefix(Properties properties, Map<String, Object> mergedConsProdConfig, String prefix) {
|
||||
Map<String, Object> mergedConfigs = new HashMap<>();
|
||||
for (String key : mergedConsProdConfig.keySet()) {
|
||||
mergedConfigs.put(key.startsWith(prefix) ? key : prefix + key, mergedConsProdConfig.get(key));
|
||||
}
|
||||
if (!ObjectUtils.isEmpty(mergedConfigs)) {
|
||||
properties.putAll(mergedConfigs);
|
||||
}
|
||||
}
|
||||
|
||||
@Bean
|
||||
public KafkaStreamsMessageConversionDelegate messageConversionDelegate(
|
||||
@Qualifier(IntegrationContextUtils.ARGUMENT_RESOLVER_MESSAGE_CONVERTER_BEAN_NAME)
|
||||
CompositeMessageConverter compositeMessageConverter,
|
||||
SendToDlqAndContinue sendToDlqAndContinue,
|
||||
KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue,
|
||||
@Qualifier("binderConfigurationProperties") KafkaStreamsBinderConfigurationProperties binderConfigurationProperties) {
|
||||
return new KafkaStreamsMessageConversionDelegate(compositeMessageConverter, sendToDlqAndContinue,
|
||||
KafkaStreamsBindingInformationCatalogue, binderConfigurationProperties);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public KStreamBoundElementFactory kStreamBoundElementFactory(
|
||||
BindingServiceProperties bindingServiceProperties,
|
||||
KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue,
|
||||
EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler) {
|
||||
return new KStreamBoundElementFactory(bindingServiceProperties,
|
||||
KafkaStreamsBindingInformationCatalogue, encodingDecodingBindAdviceHandler);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public KTableBoundElementFactory kTableBoundElementFactory(
|
||||
BindingServiceProperties bindingServiceProperties, EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler,
|
||||
KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue) {
|
||||
return new KTableBoundElementFactory(bindingServiceProperties, encodingDecodingBindAdviceHandler, KafkaStreamsBindingInformationCatalogue);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public GlobalKTableBoundElementFactory globalKTableBoundElementFactory(
|
||||
BindingServiceProperties properties, EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler,
|
||||
KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue) {
|
||||
return new GlobalKTableBoundElementFactory(properties, encodingDecodingBindAdviceHandler, KafkaStreamsBindingInformationCatalogue);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public SendToDlqAndContinue sendToDlqAndContinue() {
|
||||
return new SendToDlqAndContinue();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue() {
|
||||
return new KafkaStreamsBindingInformationCatalogue();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@SuppressWarnings("unchecked")
|
||||
@ConditionalOnMissingBean
|
||||
public KeyValueSerdeResolver keyValueSerdeResolver(
|
||||
@Qualifier("streamConfigGlobalProperties") Object streamConfigGlobalProperties,
|
||||
@Qualifier("binderConfigurationProperties")KafkaStreamsBinderConfigurationProperties properties) {
|
||||
return new KeyValueSerdeResolver(
|
||||
(Map<String, Object>) streamConfigGlobalProperties, properties);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public InteractiveQueryService interactiveQueryServices(
|
||||
KafkaStreamsRegistry kafkaStreamsRegistry,
|
||||
@Qualifier("binderConfigurationProperties")KafkaStreamsBinderConfigurationProperties properties,
|
||||
ObjectProvider<StoreQueryParametersCustomizer<?>> storeQueryParametersCustomizerProvider) {
|
||||
InteractiveQueryService interactiveQueryService = new InteractiveQueryService(kafkaStreamsRegistry, properties);
|
||||
StoreQueryParametersCustomizer<?> storeQueryParametersCustomizer = storeQueryParametersCustomizerProvider.getIfUnique();
|
||||
if (storeQueryParametersCustomizer != null) {
|
||||
interactiveQueryService.setStoreQueryParametersCustomizer(storeQueryParametersCustomizer);
|
||||
}
|
||||
return interactiveQueryService;
|
||||
}
|
||||
|
||||
@Bean
|
||||
public KafkaStreamsRegistry kafkaStreamsRegistry() {
|
||||
return new KafkaStreamsRegistry();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public StreamsBuilderFactoryManager streamsBuilderFactoryManager(
|
||||
KafkaStreamsBindingInformationCatalogue catalogue,
|
||||
KafkaStreamsRegistry kafkaStreamsRegistry,
|
||||
@Nullable KafkaStreamsBinderMetrics kafkaStreamsBinderMetrics,
|
||||
@Nullable KafkaStreamsMicrometerListener listener, KafkaProperties kafkaProperties) {
|
||||
return new StreamsBuilderFactoryManager(catalogue, kafkaStreamsRegistry, kafkaStreamsBinderMetrics, listener, kafkaProperties);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@Conditional(FunctionDetectorCondition.class)
|
||||
public KafkaStreamsFunctionProcessor kafkaStreamsFunctionProcessor(BindingServiceProperties bindingServiceProperties,
|
||||
KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties,
|
||||
KeyValueSerdeResolver keyValueSerdeResolver,
|
||||
KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue,
|
||||
KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate,
|
||||
ObjectProvider<CleanupConfig> cleanupConfig,
|
||||
StreamFunctionProperties streamFunctionProperties,
|
||||
@Qualifier("binderConfigurationProperties") KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties,
|
||||
ObjectProvider<StreamsBuilderFactoryBeanConfigurer> customizerProvider, ConfigurableEnvironment environment) {
|
||||
return new KafkaStreamsFunctionProcessor(bindingServiceProperties, kafkaStreamsExtendedBindingProperties,
|
||||
keyValueSerdeResolver, kafkaStreamsBindingInformationCatalogue, kafkaStreamsMessageConversionDelegate,
|
||||
cleanupConfig.getIfUnique(), streamFunctionProperties, kafkaStreamsBinderConfigurationProperties,
|
||||
customizerProvider.getIfUnique(), environment);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler() {
|
||||
return new EncodingDecodingBindAdviceHandler();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public DltPublishingContext dltSenderContext() {
|
||||
return new DltPublishingContext();
|
||||
}
|
||||
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@ConditionalOnMissingBean(value = KafkaStreamsBinderMetrics.class, name = "outerContext")
|
||||
@ConditionalOnClass(name = "io.micrometer.core.instrument.MeterRegistry")
|
||||
protected class KafkaStreamsBinderMetricsConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnBean(MeterRegistry.class)
|
||||
@ConditionalOnMissingBean(KafkaStreamsBinderMetrics.class)
|
||||
@ConditionalOnMissingClass("org.springframework.kafka.core.MicrometerConsumerListener")
|
||||
public KafkaStreamsBinderMetrics kafkaStreamsBinderMetrics(MeterRegistry meterRegistry) {
|
||||
|
||||
return new KafkaStreamsBinderMetrics(meterRegistry);
|
||||
}
|
||||
|
||||
@ConditionalOnClass(name = "org.springframework.kafka.core.MicrometerConsumerListener")
|
||||
@ConditionalOnBean(MeterRegistry.class)
|
||||
protected class KafkaMicrometer {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean(name = "binderStreamsListener")
|
||||
public KafkaStreamsMicrometerListener binderStreamsListener(MeterRegistry meterRegistry) {
|
||||
return new KafkaStreamsMicrometerListener(meterRegistry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@ConditionalOnBean(name = "outerContext")
|
||||
@ConditionalOnMissingBean(KafkaStreamsBinderMetrics.class)
|
||||
@ConditionalOnClass(name = "io.micrometer.core.instrument.MeterRegistry")
|
||||
protected class KafkaStreamsBinderMetricsConfigurationWithMultiBinder {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingClass("org.springframework.kafka.core.MicrometerConsumerListener")
|
||||
public KafkaStreamsBinderMetrics kafkaStreamsBinderMetrics(ConfigurableApplicationContext context) {
|
||||
|
||||
MeterRegistry meterRegistry = context.getBean("outerContext", ApplicationContext.class)
|
||||
.getBean(MeterRegistry.class);
|
||||
return new KafkaStreamsBinderMetrics(meterRegistry);
|
||||
}
|
||||
|
||||
@ConditionalOnClass(name = "org.springframework.kafka.core.MicrometerConsumerListener")
|
||||
@ConditionalOnBean(MeterRegistry.class)
|
||||
protected class KafkaMicrometer {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean(name = "binderStreamsListener")
|
||||
public KafkaStreamsMicrometerListener binderStreamsListener(MeterRegistry meterRegistry) {
|
||||
return new KafkaStreamsMicrometerListener(meterRegistry);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,251 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.function.BiFunction;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
import org.apache.kafka.clients.producer.ProducerConfig;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.kafka.common.serialization.ByteArraySerializer;
|
||||
import org.apache.kafka.streams.kstream.KStream;
|
||||
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.beans.factory.config.BeanDefinition;
|
||||
import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
|
||||
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
|
||||
import org.springframework.beans.factory.support.BeanDefinitionRegistry;
|
||||
import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.utils.DlqDestinationResolver;
|
||||
import org.springframework.cloud.stream.binder.kafka.utils.DlqPartitionFunction;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.core.MethodParameter;
|
||||
import org.springframework.kafka.config.StreamsBuilderFactoryBean;
|
||||
import org.springframework.kafka.core.DefaultKafkaProducerFactory;
|
||||
import org.springframework.kafka.core.KafkaOperations;
|
||||
import org.springframework.kafka.core.KafkaTemplate;
|
||||
import org.springframework.kafka.core.ProducerFactory;
|
||||
import org.springframework.kafka.listener.DeadLetterPublishingRecoverer;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.ObjectUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* Common methods used by various Kafka Streams types across the binders.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @author Gary Russell
|
||||
* @author James Forward
|
||||
*/
|
||||
public final class KafkaStreamsBinderUtils {
|
||||
|
||||
private static final Log LOGGER = LogFactory.getLog(KafkaStreamsBinderUtils.class);
|
||||
|
||||
private KafkaStreamsBinderUtils() {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility method to find the method targeted by the key.
|
||||
*
|
||||
* @param key name of the method
|
||||
* @param methods collection of methods to search from
|
||||
* @return found method as an {@link Optional}
|
||||
*/
|
||||
public static Optional<Method> findMethodWithName(String key, Method[] methods) {
|
||||
return Arrays.stream(methods).filter(m -> m.getName().equals(key)).findFirst();
|
||||
}
|
||||
|
||||
public static String[] deriveFunctionUnits(String definition) {
|
||||
if (!StringUtils.hasText(definition)) {
|
||||
return new String[]{};
|
||||
}
|
||||
final String[] rawSplitDefinition = definition.split(";");
|
||||
return Arrays.stream(rawSplitDefinition).map(String::trim).toArray(String[]::new);
|
||||
}
|
||||
|
||||
static void prepareConsumerBinding(String name, String group,
|
||||
ApplicationContext context, KafkaTopicProvisioner kafkaTopicProvisioner,
|
||||
KafkaStreamsBinderConfigurationProperties binderConfigurationProperties,
|
||||
ExtendedConsumerProperties<KafkaStreamsConsumerProperties> properties,
|
||||
RetryTemplate retryTemplate,
|
||||
ConfigurableListableBeanFactory beanFactory, String bindingName,
|
||||
KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue,
|
||||
StreamsBuilderFactoryBean streamsBuilderFactoryBean) {
|
||||
|
||||
ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties =
|
||||
(ExtendedConsumerProperties) properties;
|
||||
|
||||
if (binderConfigurationProperties
|
||||
.getDeserializationExceptionHandler() == DeserializationExceptionHandler.sendToDlq) {
|
||||
extendedConsumerProperties.getExtension().setEnableDlq(true);
|
||||
}
|
||||
// check for deserialization handler at the consumer binding, as that takes precedence.
|
||||
final DeserializationExceptionHandler deserializationExceptionHandler =
|
||||
properties.getExtension().getDeserializationExceptionHandler();
|
||||
if (deserializationExceptionHandler == DeserializationExceptionHandler.sendToDlq) {
|
||||
extendedConsumerProperties.getExtension().setEnableDlq(true);
|
||||
}
|
||||
|
||||
String[] inputTopics = StringUtils.commaDelimitedListToStringArray(name);
|
||||
for (String inputTopic : inputTopics) {
|
||||
kafkaTopicProvisioner.provisionConsumerDestination(inputTopic, group,
|
||||
extendedConsumerProperties);
|
||||
}
|
||||
|
||||
if (extendedConsumerProperties.getExtension().isEnableDlq()) {
|
||||
|
||||
Map<String, DlqPartitionFunction> partitionFunctions =
|
||||
context.getBeansOfType(DlqPartitionFunction.class, false, false);
|
||||
boolean oneFunctionPresent = partitionFunctions.size() == 1;
|
||||
Integer dlqPartitions = extendedConsumerProperties.getExtension().getDlqPartitions();
|
||||
DlqPartitionFunction partitionFunction = oneFunctionPresent
|
||||
? partitionFunctions.values().iterator().next()
|
||||
: DlqPartitionFunction.determineFallbackFunction(dlqPartitions, LOGGER);
|
||||
|
||||
ProducerFactory<byte[], byte[]> producerFactory = getProducerFactory(
|
||||
new ExtendedProducerProperties<>(
|
||||
extendedConsumerProperties.getExtension().getDlqProducerProperties()),
|
||||
binderConfigurationProperties);
|
||||
kafkaStreamsBindingInformationCatalogue.addDlqProducerFactory(streamsBuilderFactoryBean, producerFactory);
|
||||
|
||||
KafkaOperations<byte[], byte[]> kafkaTemplate = new KafkaTemplate<>(producerFactory);
|
||||
|
||||
Map<String, DlqDestinationResolver> dlqDestinationResolvers =
|
||||
context.getBeansOfType(DlqDestinationResolver.class, false, false);
|
||||
|
||||
BiFunction<ConsumerRecord<?, ?>, Exception, TopicPartition> destinationResolver =
|
||||
dlqDestinationResolvers.isEmpty() ? (cr, e) -> new TopicPartition(extendedConsumerProperties.getExtension().getDlqName(),
|
||||
partitionFunction.apply(group, cr, e)) :
|
||||
(cr, e) -> new TopicPartition(dlqDestinationResolvers.values().iterator().next().apply(cr, e),
|
||||
partitionFunction.apply(group, cr, e));
|
||||
|
||||
DeadLetterPublishingRecoverer kafkaStreamsBinderDlqRecoverer = !dlqDestinationResolvers.isEmpty() || StringUtils
|
||||
.hasText(extendedConsumerProperties.getExtension().getDlqName())
|
||||
? new DeadLetterPublishingRecoverer(kafkaTemplate, destinationResolver)
|
||||
: null;
|
||||
for (String inputTopic : inputTopics) {
|
||||
if (!StringUtils.hasText(
|
||||
extendedConsumerProperties.getExtension().getDlqName()) && dlqDestinationResolvers.isEmpty()) {
|
||||
destinationResolver = (cr, e) -> new TopicPartition("error." + inputTopic + "." + group,
|
||||
partitionFunction.apply(group, cr, e));
|
||||
kafkaStreamsBinderDlqRecoverer = new DeadLetterPublishingRecoverer(kafkaTemplate,
|
||||
destinationResolver);
|
||||
}
|
||||
SendToDlqAndContinue sendToDlqAndContinue = context
|
||||
.getBean(SendToDlqAndContinue.class);
|
||||
sendToDlqAndContinue.addKStreamDlqDispatch(inputTopic,
|
||||
kafkaStreamsBinderDlqRecoverer);
|
||||
}
|
||||
}
|
||||
|
||||
if (!StringUtils.hasText(properties.getRetryTemplateName())) {
|
||||
@SuppressWarnings("unchecked")
|
||||
BeanDefinition retryTemplateBeanDefinition = BeanDefinitionBuilder
|
||||
.genericBeanDefinition(
|
||||
(Class<RetryTemplate>) retryTemplate.getClass(),
|
||||
() -> retryTemplate)
|
||||
.getRawBeanDefinition();
|
||||
((BeanDefinitionRegistry) beanFactory).registerBeanDefinition(bindingName + "-RetryTemplate", retryTemplateBeanDefinition);
|
||||
}
|
||||
}
|
||||
|
||||
private static DefaultKafkaProducerFactory<byte[], byte[]> getProducerFactory(
|
||||
ExtendedProducerProperties<KafkaProducerProperties> producerProperties,
|
||||
KafkaBinderConfigurationProperties configurationProperties) {
|
||||
Map<String, Object> props = new HashMap<>();
|
||||
props.put(ProducerConfig.RETRIES_CONFIG, 0);
|
||||
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432);
|
||||
props.put(ProducerConfig.ACKS_CONFIG, configurationProperties.getRequiredAcks());
|
||||
Map<String, Object> mergedConfig = configurationProperties
|
||||
.mergedProducerConfiguration();
|
||||
if (!ObjectUtils.isEmpty(mergedConfig)) {
|
||||
props.putAll(mergedConfig);
|
||||
}
|
||||
if (ObjectUtils.isEmpty(props.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG))) {
|
||||
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,
|
||||
configurationProperties.getKafkaConnectionString());
|
||||
}
|
||||
if (ObjectUtils.isEmpty(props.get(ProducerConfig.BATCH_SIZE_CONFIG))) {
|
||||
props.put(ProducerConfig.BATCH_SIZE_CONFIG,
|
||||
String.valueOf(producerProperties.getExtension().getBufferSize()));
|
||||
}
|
||||
if (ObjectUtils.isEmpty(props.get(ProducerConfig.LINGER_MS_CONFIG))) {
|
||||
props.put(ProducerConfig.LINGER_MS_CONFIG,
|
||||
String.valueOf(producerProperties.getExtension().getBatchTimeout()));
|
||||
}
|
||||
if (ObjectUtils.isEmpty(props.get(ProducerConfig.COMPRESSION_TYPE_CONFIG))) {
|
||||
props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG,
|
||||
producerProperties.getExtension().getCompressionType().toString());
|
||||
}
|
||||
Map<String, String> configs = producerProperties.getExtension().getConfiguration();
|
||||
Assert.state(!configs.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG),
|
||||
ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " cannot be overridden at the binding level; "
|
||||
+ "use multiple binders instead");
|
||||
if (!ObjectUtils.isEmpty(configs)) {
|
||||
props.putAll(configs);
|
||||
}
|
||||
// Always send as byte[] on dlq (the same byte[] that the consumer received)
|
||||
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class);
|
||||
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
|
||||
ByteArraySerializer.class);
|
||||
|
||||
return new DefaultKafkaProducerFactory<>(props);
|
||||
}
|
||||
|
||||
|
||||
static boolean supportsKStream(MethodParameter methodParameter, Class<?> targetBeanClass) {
|
||||
return KStream.class.isAssignableFrom(targetBeanClass)
|
||||
&& KStream.class.isAssignableFrom(methodParameter.getParameterType());
|
||||
}
|
||||
|
||||
static void closeDlqProducerFactories(KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue,
|
||||
StreamsBuilderFactoryBean streamsBuilderFactoryBean) {
|
||||
|
||||
final List<ProducerFactory<byte[], byte[]>> dlqProducerFactories =
|
||||
kafkaStreamsBindingInformationCatalogue.getDlqProducerFactory(streamsBuilderFactoryBean);
|
||||
|
||||
if (!CollectionUtils.isEmpty(dlqProducerFactories)) {
|
||||
for (ProducerFactory<byte[], byte[]> producerFactory : dlqProducerFactories) {
|
||||
try {
|
||||
((DisposableBean) producerFactory).destroy();
|
||||
}
|
||||
catch (Exception exception) {
|
||||
throw new IllegalStateException(exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,250 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2021 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.kafka.common.serialization.Serde;
|
||||
import org.apache.kafka.streams.KafkaStreams;
|
||||
import org.apache.kafka.streams.StreamsConfig;
|
||||
import org.apache.kafka.streams.kstream.KStream;
|
||||
|
||||
import org.springframework.cloud.stream.binder.ConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties;
|
||||
import org.springframework.cloud.stream.config.BindingProperties;
|
||||
import org.springframework.core.ResolvableType;
|
||||
import org.springframework.kafka.config.StreamsBuilderFactoryBean;
|
||||
import org.springframework.kafka.core.ProducerFactory;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
/**
|
||||
* A catalogue that provides binding information for Kafka Streams target types such as
|
||||
* KStream. It also keeps a catalogue for the underlying {@link StreamsBuilderFactoryBean}
|
||||
* and {@link StreamsConfig} associated with various
|
||||
* Kafka Streams functions in the
|
||||
* {@link org.springframework.context.ApplicationContext}.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
public class KafkaStreamsBindingInformationCatalogue {
|
||||
|
||||
private final Map<KStream<?, ?>, BindingProperties> bindingProperties = new ConcurrentHashMap<>();
|
||||
|
||||
private final Map<KStream<?, ?>, KafkaStreamsConsumerProperties> consumerProperties = new ConcurrentHashMap<>();
|
||||
|
||||
private final Map<String, StreamsBuilderFactoryBean> streamsBuilderFactoryBeanPerBinding = new HashMap<>();
|
||||
|
||||
private final Map<StreamsBuilderFactoryBean, List<ConsumerProperties>> consumerPropertiesPerSbfb = new HashMap<>();
|
||||
|
||||
private final Map<Object, ResolvableType> outboundKStreamResolvables = new HashMap<>();
|
||||
|
||||
private final Map<KStream<?, ?>, Serde<?>> keySerdeInfo = new HashMap<>();
|
||||
|
||||
private final Map<Object, String> bindingNamesPerTarget = new HashMap<>();
|
||||
|
||||
private final Map<String, KafkaStreams> previousKafkaStreamsPerApplicationId = new HashMap<>();
|
||||
|
||||
private final Map<StreamsBuilderFactoryBean, List<ProducerFactory<byte[], byte[]>>> dlqProducerFactories = new HashMap<>();
|
||||
|
||||
/**
|
||||
* For a given bounded {@link KStream}, retrieve it's corresponding destination on the
|
||||
* broker.
|
||||
* @param bindingTarget binding target for KStream
|
||||
* @return destination topic on Kafka
|
||||
*/
|
||||
String getDestination(KStream<?, ?> bindingTarget) {
|
||||
BindingProperties bindingProperties = this.bindingProperties.get(bindingTarget);
|
||||
return bindingProperties.getDestination();
|
||||
}
|
||||
|
||||
/**
|
||||
* Is native decoding is enabled on this {@link KStream}.
|
||||
* @param bindingTarget binding target for KStream
|
||||
* @return true if native decoding is enabled, fasle otherwise.
|
||||
*/
|
||||
boolean isUseNativeDecoding(KStream<?, ?> bindingTarget) {
|
||||
BindingProperties bindingProperties = this.bindingProperties.get(bindingTarget);
|
||||
if (bindingProperties.getConsumer() == null) {
|
||||
bindingProperties.setConsumer(new ConsumerProperties());
|
||||
}
|
||||
return bindingProperties.getConsumer().isUseNativeDecoding();
|
||||
}
|
||||
|
||||
/**
|
||||
* Is DLQ enabled for this {@link KStream}.
|
||||
* @param bindingTarget binding target for KStream
|
||||
* @return true if DLQ is enabled, false otherwise.
|
||||
*/
|
||||
boolean isDlqEnabled(KStream<?, ?> bindingTarget) {
|
||||
return this.consumerProperties.get(bindingTarget).isEnableDlq();
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve the content type associated with a given {@link KStream}.
|
||||
* @param bindingTarget binding target for KStream
|
||||
* @return content Type associated.
|
||||
*/
|
||||
String getContentType(KStream<?, ?> bindingTarget) {
|
||||
BindingProperties bindingProperties = this.bindingProperties.get(bindingTarget);
|
||||
return bindingProperties.getContentType();
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a cache for bounded KStream -> {@link BindingProperties}.
|
||||
* @param bindingTarget binding target for KStream
|
||||
* @param bindingProperties {@link BindingProperties} for this KStream
|
||||
*/
|
||||
void registerBindingProperties(KStream<?, ?> bindingTarget,
|
||||
BindingProperties bindingProperties) {
|
||||
if (bindingProperties != null) {
|
||||
this.bindingProperties.put(bindingTarget, bindingProperties);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a cache for bounded KStream -> {@link KafkaStreamsConsumerProperties}.
|
||||
* @param bindingTarget binding target for KStream
|
||||
* @param kafkaStreamsConsumerProperties consumer properties for this KStream
|
||||
*/
|
||||
void registerConsumerProperties(KStream<?, ?> bindingTarget,
|
||||
KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties) {
|
||||
if (kafkaStreamsConsumerProperties != null) {
|
||||
this.consumerProperties.put(bindingTarget, kafkaStreamsConsumerProperties);
|
||||
}
|
||||
}
|
||||
|
||||
Set<StreamsBuilderFactoryBean> getStreamsBuilderFactoryBeans() {
|
||||
return new HashSet<>(this.streamsBuilderFactoryBeanPerBinding.values());
|
||||
}
|
||||
|
||||
void addStreamBuilderFactoryPerBinding(String binding, StreamsBuilderFactoryBean streamsBuilderFactoryBean) {
|
||||
this.streamsBuilderFactoryBeanPerBinding.put(binding, streamsBuilderFactoryBean);
|
||||
}
|
||||
|
||||
void addConsumerPropertiesPerSbfb(StreamsBuilderFactoryBean streamsBuilderFactoryBean, ConsumerProperties consumerProperties) {
|
||||
this.consumerPropertiesPerSbfb.computeIfAbsent(streamsBuilderFactoryBean, k -> new ArrayList<>());
|
||||
this.consumerPropertiesPerSbfb.get(streamsBuilderFactoryBean).add(consumerProperties);
|
||||
}
|
||||
|
||||
public Map<StreamsBuilderFactoryBean, List<ConsumerProperties>> getConsumerPropertiesPerSbfb() {
|
||||
return this.consumerPropertiesPerSbfb;
|
||||
}
|
||||
|
||||
Map<String, StreamsBuilderFactoryBean> getStreamsBuilderFactoryBeanPerBinding() {
|
||||
return this.streamsBuilderFactoryBeanPerBinding;
|
||||
}
|
||||
|
||||
void addOutboundKStreamResolvable(Object key, ResolvableType outboundResolvable) {
|
||||
this.outboundKStreamResolvables.put(key, outboundResolvable);
|
||||
}
|
||||
|
||||
ResolvableType getOutboundKStreamResolvable(Object key) {
|
||||
return outboundKStreamResolvables.get(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adding a mapping for KStream target to its corresponding KeySerde.
|
||||
* This is used for sending to DLQ when deserialization fails. See {@link KafkaStreamsMessageConversionDelegate}
|
||||
* for details.
|
||||
*
|
||||
* @param kStreamTarget target KStream
|
||||
* @param keySerde Serde used for the key
|
||||
*/
|
||||
void addKeySerde(KStream<?, ?> kStreamTarget, Serde<?> keySerde) {
|
||||
this.keySerdeInfo.put(kStreamTarget, keySerde);
|
||||
}
|
||||
|
||||
Serde<?> getKeySerde(KStream<?, ?> kStreamTarget) {
|
||||
return this.keySerdeInfo.get(kStreamTarget);
|
||||
}
|
||||
|
||||
|
||||
Map<KStream<?, ?>, BindingProperties> getBindingProperties() {
|
||||
return bindingProperties;
|
||||
}
|
||||
|
||||
Map<KStream<?, ?>, KafkaStreamsConsumerProperties> getConsumerProperties() {
|
||||
return consumerProperties;
|
||||
}
|
||||
|
||||
void addBindingNamePerTarget(Object target, String bindingName) {
|
||||
this.bindingNamesPerTarget.put(target, bindingName);
|
||||
}
|
||||
|
||||
String bindingNamePerTarget(Object target) {
|
||||
return this.bindingNamesPerTarget.get(target);
|
||||
}
|
||||
|
||||
public List<ProducerFactory<byte[], byte[]>> getDlqProducerFactories() {
|
||||
return this.dlqProducerFactories.values()
|
||||
.stream()
|
||||
.flatMap(List::stream)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public List<ProducerFactory<byte[], byte[]>> getDlqProducerFactory(StreamsBuilderFactoryBean streamsBuilderFactoryBean) {
|
||||
return this.dlqProducerFactories.get(streamsBuilderFactoryBean);
|
||||
}
|
||||
|
||||
public void addDlqProducerFactory(StreamsBuilderFactoryBean streamsBuilderFactoryBean,
|
||||
ProducerFactory<byte[], byte[]> producerFactory) {
|
||||
List<ProducerFactory<byte[], byte[]>> producerFactories = this.dlqProducerFactories.get(streamsBuilderFactoryBean);
|
||||
if (CollectionUtils.isEmpty(producerFactories)) {
|
||||
producerFactories = new ArrayList<>();
|
||||
this.dlqProducerFactories.put(streamsBuilderFactoryBean, producerFactories);
|
||||
}
|
||||
producerFactories.add(producerFactory);
|
||||
}
|
||||
|
||||
/**
|
||||
* Caching the previous KafkaStreams for the application.id when binding is stopped through actuator.
|
||||
* See https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165
|
||||
*
|
||||
* @param applicationId application.id
|
||||
* @param kafkaStreams {@link KafkaStreams} object
|
||||
*/
|
||||
public void addPreviousKafkaStreamsForApplicationId(String applicationId, KafkaStreams kafkaStreams) {
|
||||
this.previousKafkaStreamsPerApplicationId.put(applicationId, kafkaStreams);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove the previously cached KafkaStreams object.
|
||||
* See https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165
|
||||
*
|
||||
* @param applicationId application.id
|
||||
*/
|
||||
public void removePreviousKafkaStreamsForApplicationId(String applicationId) {
|
||||
this.previousKafkaStreamsPerApplicationId.remove(applicationId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all stopped KafkaStreams objects through actuator binding stop.
|
||||
* See https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/issues/1165
|
||||
*
|
||||
* @return stopped KafkaStreams objects map
|
||||
*/
|
||||
public Map<String, KafkaStreams> getStoppedKafkaStreams() {
|
||||
return this.previousKafkaStreamsPerApplicationId;
|
||||
}
|
||||
}
|
||||
@@ -1,572 +0,0 @@
|
||||
/*
|
||||
* Copyright 2019-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.BiFunction;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.kafka.common.serialization.Serde;
|
||||
import org.apache.kafka.common.serialization.Serdes;
|
||||
import org.apache.kafka.streams.StreamsBuilder;
|
||||
import org.apache.kafka.streams.StreamsConfig;
|
||||
import org.apache.kafka.streams.Topology;
|
||||
import org.apache.kafka.streams.kstream.KStream;
|
||||
import org.apache.kafka.streams.kstream.KTable;
|
||||
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.beans.factory.BeanFactory;
|
||||
import org.springframework.beans.factory.BeanFactoryAware;
|
||||
import org.springframework.beans.factory.BeanInitializationException;
|
||||
import org.springframework.beans.factory.support.BeanDefinitionRegistry;
|
||||
import org.springframework.beans.factory.support.RootBeanDefinition;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.function.KafkaStreamsBindableProxyFactory;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties;
|
||||
import org.springframework.cloud.stream.config.BindingProperties;
|
||||
import org.springframework.cloud.stream.config.BindingServiceProperties;
|
||||
import org.springframework.cloud.stream.function.FunctionConstants;
|
||||
import org.springframework.cloud.stream.function.StreamFunctionProperties;
|
||||
import org.springframework.core.ResolvableType;
|
||||
import org.springframework.core.env.ConfigurableEnvironment;
|
||||
import org.springframework.kafka.config.StreamsBuilderFactoryBean;
|
||||
import org.springframework.kafka.config.StreamsBuilderFactoryBeanConfigurer;
|
||||
import org.springframework.kafka.core.CleanupConfig;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
/**
|
||||
* @author Soby Chacko
|
||||
* @author Byungjun You
|
||||
* @since 2.2.0
|
||||
*/
|
||||
public class KafkaStreamsFunctionProcessor extends AbstractKafkaStreamsBinderProcessor implements BeanFactoryAware {
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(KafkaStreamsFunctionProcessor.class);
|
||||
private static final String OUTBOUND = "outbound";
|
||||
|
||||
private final BindingServiceProperties bindingServiceProperties;
|
||||
private final Map<String, StreamsBuilderFactoryBean> methodStreamsBuilderFactoryBeanMap = new HashMap<>();
|
||||
private final KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties;
|
||||
private final KeyValueSerdeResolver keyValueSerdeResolver;
|
||||
private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue;
|
||||
private final KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate;
|
||||
|
||||
private BeanFactory beanFactory;
|
||||
private StreamFunctionProperties streamFunctionProperties;
|
||||
private KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties;
|
||||
StreamsBuilderFactoryBeanConfigurer customizer;
|
||||
ConfigurableEnvironment environment;
|
||||
|
||||
public KafkaStreamsFunctionProcessor(BindingServiceProperties bindingServiceProperties,
|
||||
KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties,
|
||||
KeyValueSerdeResolver keyValueSerdeResolver,
|
||||
KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue,
|
||||
KafkaStreamsMessageConversionDelegate kafkaStreamsMessageConversionDelegate,
|
||||
CleanupConfig cleanupConfig,
|
||||
StreamFunctionProperties streamFunctionProperties,
|
||||
KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties,
|
||||
StreamsBuilderFactoryBeanConfigurer customizer, ConfigurableEnvironment environment) {
|
||||
super(bindingServiceProperties, kafkaStreamsBindingInformationCatalogue, kafkaStreamsExtendedBindingProperties,
|
||||
keyValueSerdeResolver, cleanupConfig);
|
||||
this.bindingServiceProperties = bindingServiceProperties;
|
||||
this.kafkaStreamsExtendedBindingProperties = kafkaStreamsExtendedBindingProperties;
|
||||
this.keyValueSerdeResolver = keyValueSerdeResolver;
|
||||
this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue;
|
||||
this.kafkaStreamsMessageConversionDelegate = kafkaStreamsMessageConversionDelegate;
|
||||
this.streamFunctionProperties = streamFunctionProperties;
|
||||
this.kafkaStreamsBinderConfigurationProperties = kafkaStreamsBinderConfigurationProperties;
|
||||
this.customizer = customizer;
|
||||
this.environment = environment;
|
||||
}
|
||||
|
||||
private Map<String, ResolvableType> buildTypeMap(ResolvableType resolvableType,
|
||||
KafkaStreamsBindableProxyFactory kafkaStreamsBindableProxyFactory,
|
||||
Method method, String functionName) {
|
||||
Map<String, ResolvableType> resolvableTypeMap = new LinkedHashMap<>();
|
||||
if (method != null) { // Component functional bean.
|
||||
final ResolvableType firstMethodParameter = ResolvableType.forMethodParameter(method, 0);
|
||||
ResolvableType currentOutputGeneric = ResolvableType.forMethodReturnType(method);
|
||||
|
||||
final Set<String> inputs = new LinkedHashSet<>(kafkaStreamsBindableProxyFactory.getInputs());
|
||||
final Iterator<String> iterator = inputs.iterator();
|
||||
populateResolvableTypeMap(firstMethodParameter, resolvableTypeMap, iterator, method, functionName);
|
||||
|
||||
final Class<?> outputRawclass = currentOutputGeneric.getRawClass();
|
||||
traverseReturnTypeForComponentBeans(resolvableTypeMap, currentOutputGeneric, inputs, iterator, outputRawclass);
|
||||
}
|
||||
else if (resolvableType != null && resolvableType.getRawClass() != null) {
|
||||
int inputCount = 1;
|
||||
|
||||
ResolvableType currentOutputGeneric;
|
||||
if (resolvableType.getRawClass().isAssignableFrom(BiFunction.class) ||
|
||||
resolvableType.getRawClass().isAssignableFrom(BiConsumer.class)) {
|
||||
inputCount = 2;
|
||||
currentOutputGeneric = resolvableType.getGeneric(2);
|
||||
}
|
||||
else {
|
||||
currentOutputGeneric = resolvableType.getGeneric(1);
|
||||
}
|
||||
while (currentOutputGeneric.getRawClass() != null && functionOrConsumerFound(currentOutputGeneric)) {
|
||||
inputCount++;
|
||||
currentOutputGeneric = currentOutputGeneric.getGeneric(1);
|
||||
}
|
||||
final Set<String> inputs = new LinkedHashSet<>(kafkaStreamsBindableProxyFactory.getInputs());
|
||||
|
||||
final Iterator<String> iterator = inputs.iterator();
|
||||
|
||||
populateResolvableTypeMap(resolvableType, resolvableTypeMap, iterator);
|
||||
|
||||
ResolvableType iterableResType = resolvableType;
|
||||
int i = resolvableType.getRawClass().isAssignableFrom(BiFunction.class) ||
|
||||
resolvableType.getRawClass().isAssignableFrom(BiConsumer.class) ? 2 : 1;
|
||||
ResolvableType outboundResolvableType;
|
||||
if (i == inputCount) {
|
||||
outboundResolvableType = iterableResType.getGeneric(i);
|
||||
}
|
||||
else {
|
||||
while (i < inputCount && iterator.hasNext()) {
|
||||
iterableResType = iterableResType.getGeneric(1);
|
||||
if (iterableResType.getRawClass() != null &&
|
||||
functionOrConsumerFound(iterableResType)) {
|
||||
populateResolvableTypeMap(iterableResType, resolvableTypeMap, iterator);
|
||||
}
|
||||
i++;
|
||||
}
|
||||
outboundResolvableType = iterableResType.getGeneric(1);
|
||||
}
|
||||
resolvableTypeMap.put(OUTBOUND, outboundResolvableType);
|
||||
}
|
||||
return resolvableTypeMap;
|
||||
}
|
||||
|
||||
private void traverseReturnTypeForComponentBeans(Map<String, ResolvableType> resolvableTypeMap, ResolvableType currentOutputGeneric,
|
||||
Set<String> inputs, Iterator<String> iterator, Class<?> outputRawclass) {
|
||||
if (outputRawclass != null && !outputRawclass.equals(Void.TYPE)) {
|
||||
ResolvableType iterableResType = currentOutputGeneric;
|
||||
int i = 1;
|
||||
// Traverse through the return signature.
|
||||
while (i < inputs.size() && iterator.hasNext()) {
|
||||
if (iterableResType.getRawClass() != null &&
|
||||
functionOrConsumerFound(iterableResType)) {
|
||||
populateResolvableTypeMap(iterableResType, resolvableTypeMap, iterator);
|
||||
}
|
||||
iterableResType = iterableResType.getGeneric(1);
|
||||
i++;
|
||||
}
|
||||
if (iterableResType.getRawClass() != null && KStream.class.isAssignableFrom(iterableResType.getRawClass())) {
|
||||
resolvableTypeMap.put(OUTBOUND, iterableResType);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean functionOrConsumerFound(ResolvableType iterableResType) {
|
||||
return iterableResType.getRawClass().equals(Function.class) ||
|
||||
iterableResType.getRawClass().equals(Consumer.class);
|
||||
}
|
||||
|
||||
private void populateResolvableTypeMap(ResolvableType resolvableType, Map<String, ResolvableType> resolvableTypeMap,
|
||||
Iterator<String> iterator) {
|
||||
final String next = iterator.next();
|
||||
resolvableTypeMap.put(next, resolvableType.getGeneric(0));
|
||||
if (resolvableType.getRawClass() != null &&
|
||||
(resolvableType.getRawClass().isAssignableFrom(BiFunction.class) ||
|
||||
resolvableType.getRawClass().isAssignableFrom(BiConsumer.class))
|
||||
&& iterator.hasNext()) {
|
||||
resolvableTypeMap.put(iterator.next(), resolvableType.getGeneric(1));
|
||||
}
|
||||
}
|
||||
|
||||
private void populateResolvableTypeMap(ResolvableType resolvableType, Map<String, ResolvableType> resolvableTypeMap,
|
||||
Iterator<String> iterator, Method method, String functionName) {
|
||||
final String next = iterator.next();
|
||||
resolvableTypeMap.put(next, resolvableType);
|
||||
if (method != null) {
|
||||
final Object bean = beanFactory.getBean(functionName);
|
||||
if (BiFunction.class.isAssignableFrom(bean.getClass()) || BiConsumer.class.isAssignableFrom(bean.getClass())) {
|
||||
resolvableTypeMap.put(iterator.next(), ResolvableType.forMethodParameter(method, 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private ResolvableType checkOutboundForComposedFunctions(
|
||||
ResolvableType outputResolvableType) {
|
||||
|
||||
ResolvableType currentOutputGeneric;
|
||||
|
||||
if (outputResolvableType.getRawClass() != null && outputResolvableType.getRawClass().isAssignableFrom(BiFunction.class)) {
|
||||
currentOutputGeneric = outputResolvableType.getGeneric(2);
|
||||
}
|
||||
else {
|
||||
currentOutputGeneric = outputResolvableType.getGeneric(1);
|
||||
}
|
||||
while (currentOutputGeneric.getRawClass() != null && functionOrConsumerFound(currentOutputGeneric)) {
|
||||
currentOutputGeneric = currentOutputGeneric.getGeneric(1);
|
||||
}
|
||||
return currentOutputGeneric;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method must be kept stateless. In the case of multiple function beans in an application,
|
||||
* isolated {@link KafkaStreamsBindableProxyFactory} instances are passed in separately for those functions. If the
|
||||
* state is shared between invocations, that will create potential race conditions. Hence, invocations of this method
|
||||
* should not be dependent on state modified by a previous invocation.
|
||||
*
|
||||
* @param resolvableType type of the binding
|
||||
* @param functionName bean name of the function
|
||||
* @param kafkaStreamsBindableProxyFactory bindable proxy factory for the Kafka Streams type
|
||||
*/
|
||||
@SuppressWarnings({ "unchecked", "rawtypes" })
|
||||
public void setupFunctionInvokerForKafkaStreams(ResolvableType resolvableType, String functionName,
|
||||
KafkaStreamsBindableProxyFactory kafkaStreamsBindableProxyFactory, Method method,
|
||||
ResolvableType outputResolvableType,
|
||||
String... composedFunctionNames) {
|
||||
final Map<String, ResolvableType> resolvableTypes = buildTypeMap(resolvableType,
|
||||
kafkaStreamsBindableProxyFactory, method, functionName);
|
||||
|
||||
ResolvableType outboundResolvableType;
|
||||
if (outputResolvableType != null) {
|
||||
outboundResolvableType = checkOutboundForComposedFunctions(outputResolvableType);
|
||||
resolvableTypes.remove(OUTBOUND);
|
||||
}
|
||||
else {
|
||||
outboundResolvableType = resolvableTypes.remove(OUTBOUND);
|
||||
}
|
||||
|
||||
Object[] adaptedInboundArguments = adaptAndRetrieveInboundArguments(resolvableTypes, functionName);
|
||||
try {
|
||||
if (resolvableType.getRawClass() != null && resolvableType.getRawClass().equals(Consumer.class)) {
|
||||
Consumer<Object> consumer = (Consumer) this.beanFactory.getBean(functionName);
|
||||
consumer.accept(adaptedInboundArguments[0]);
|
||||
}
|
||||
else if (resolvableType.getRawClass() != null && resolvableType.getRawClass().equals(BiConsumer.class)) {
|
||||
BiConsumer<Object, Object> biConsumer = (BiConsumer) this.beanFactory.getBean(functionName);
|
||||
biConsumer.accept(adaptedInboundArguments[0], adaptedInboundArguments[1]);
|
||||
}
|
||||
else if (method != null) { // Handling component functional beans
|
||||
final Object bean = beanFactory.getBean(functionName);
|
||||
if (Consumer.class.isAssignableFrom(bean.getClass())) {
|
||||
((Consumer) bean).accept(adaptedInboundArguments[0]);
|
||||
}
|
||||
else if (BiConsumer.class.isAssignableFrom(bean.getClass())) {
|
||||
((BiConsumer) bean).accept(adaptedInboundArguments[0], adaptedInboundArguments[1]);
|
||||
}
|
||||
else if (Function.class.isAssignableFrom(bean.getClass()) || BiFunction.class.isAssignableFrom(bean.getClass())) {
|
||||
Object result;
|
||||
if (BiFunction.class.isAssignableFrom(bean.getClass())) {
|
||||
result = ((BiFunction) bean).apply(adaptedInboundArguments[0], adaptedInboundArguments[1]);
|
||||
}
|
||||
else {
|
||||
result = ((Function) bean).apply(adaptedInboundArguments[0]);
|
||||
}
|
||||
result = handleCurriedFunctions(adaptedInboundArguments, result);
|
||||
if (result != null) {
|
||||
final Set<String> outputs = new TreeSet<>(kafkaStreamsBindableProxyFactory.getOutputs());
|
||||
final Iterator<String> outboundDefinitionIterator = outputs.iterator();
|
||||
if (result.getClass().isArray()) {
|
||||
final String initialInput = resolvableTypes.keySet().iterator().next();
|
||||
final StreamsBuilderFactoryBean streamsBuilderFactoryBean =
|
||||
this.kafkaStreamsBindingInformationCatalogue.getStreamsBuilderFactoryBeanPerBinding().get(initialInput);
|
||||
handleKStreamArrayOutbound(resolvableType, functionName, kafkaStreamsBindableProxyFactory,
|
||||
outboundResolvableType, (Object[]) result, streamsBuilderFactoryBean);
|
||||
}
|
||||
else {
|
||||
if (KTable.class.isAssignableFrom(result.getClass())) {
|
||||
handleSingleKStreamOutbound(resolvableTypes, outboundResolvableType != null ?
|
||||
outboundResolvableType : resolvableType.getGeneric(1), ((KTable) result).toStream(), outboundDefinitionIterator);
|
||||
}
|
||||
else {
|
||||
handleSingleKStreamOutbound(resolvableTypes, outboundResolvableType, (KStream) result, outboundDefinitionIterator);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
Object result = null;
|
||||
if (resolvableType.getRawClass() != null && resolvableType.getRawClass().equals(BiFunction.class)) {
|
||||
if (composedFunctionNames != null && composedFunctionNames.length > 0) {
|
||||
result = handleComposedFunctions(adaptedInboundArguments, result, composedFunctionNames);
|
||||
}
|
||||
else {
|
||||
BiFunction<Object, Object, Object> biFunction = (BiFunction) beanFactory.getBean(functionName);
|
||||
result = biFunction.apply(adaptedInboundArguments[0], adaptedInboundArguments[1]);
|
||||
result = handleCurriedFunctions(adaptedInboundArguments, result);
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (composedFunctionNames != null && composedFunctionNames.length > 0) {
|
||||
result = handleComposedFunctions(adaptedInboundArguments, result, composedFunctionNames);
|
||||
}
|
||||
else {
|
||||
Function<Object, Object> function = (Function) beanFactory.getBean(functionName);
|
||||
result = function.apply(adaptedInboundArguments[0]);
|
||||
result = handleCurriedFunctions(adaptedInboundArguments, result);
|
||||
}
|
||||
}
|
||||
if (result != null) {
|
||||
final Set<String> outputs = new TreeSet<>(kafkaStreamsBindableProxyFactory.getOutputs());
|
||||
final Iterator<String> outboundDefinitionIterator = outputs.iterator();
|
||||
|
||||
if (result.getClass().isArray()) {
|
||||
final String initialInput = resolvableTypes.keySet().iterator().next();
|
||||
final StreamsBuilderFactoryBean streamsBuilderFactoryBean =
|
||||
this.kafkaStreamsBindingInformationCatalogue.getStreamsBuilderFactoryBeanPerBinding().get(initialInput);
|
||||
handleKStreamArrayOutbound(resolvableType, functionName, kafkaStreamsBindableProxyFactory,
|
||||
outboundResolvableType, (Object[]) result, streamsBuilderFactoryBean);
|
||||
}
|
||||
else {
|
||||
if (KTable.class.isAssignableFrom(result.getClass())) {
|
||||
handleSingleKStreamOutbound(resolvableTypes, outboundResolvableType != null ?
|
||||
outboundResolvableType : resolvableType.getGeneric(1), ((KTable) result).toStream(), outboundDefinitionIterator);
|
||||
}
|
||||
else {
|
||||
handleSingleKStreamOutbound(resolvableTypes, outboundResolvableType != null ?
|
||||
outboundResolvableType : resolvableType.getGeneric(1), (KStream) result, outboundDefinitionIterator);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex) {
|
||||
throw new BeanInitializationException("Cannot setup function invoker for this Kafka Streams function.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked"})
|
||||
private Object handleComposedFunctions(Object[] adaptedInboundArguments, Object result, String... composedFunctionNames) {
|
||||
Object bean = beanFactory.getBean(composedFunctionNames[0]);
|
||||
if (BiFunction.class.isAssignableFrom(bean.getClass())) {
|
||||
result = ((BiFunction<Object, Object, Object>) bean).apply(adaptedInboundArguments[0], adaptedInboundArguments[1]);
|
||||
}
|
||||
else if (Function.class.isAssignableFrom(bean.getClass())) {
|
||||
result = ((Function<Object, Object>) bean).apply(adaptedInboundArguments[0]);
|
||||
}
|
||||
// If the return is a curried function, apply it
|
||||
result = handleCurriedFunctions(adaptedInboundArguments, result);
|
||||
// Apply composed functions
|
||||
return applyComposedFunctions(result, composedFunctionNames);
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked"})
|
||||
private Object applyComposedFunctions(Object result, String[] composedFunctionNames) {
|
||||
for (int i = 1; i < composedFunctionNames.length; i++) {
|
||||
final Object bean = beanFactory.getBean(composedFunctionNames[i]);
|
||||
if (Consumer.class.isAssignableFrom(bean.getClass())) {
|
||||
((Consumer<Object>) bean).accept(result);
|
||||
result = null;
|
||||
}
|
||||
else if (Function.class.isAssignableFrom(bean.getClass())) {
|
||||
result = ((Function<Object, Object>) bean).apply(result);
|
||||
}
|
||||
else {
|
||||
throw new IllegalStateException("You can only compose functions of type either java.util.function.Function or java.util.function.Consumer.");
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked"})
|
||||
private Object handleCurriedFunctions(Object[] adaptedInboundArguments, Object result) {
|
||||
int i = 1;
|
||||
while (result instanceof Function || result instanceof Consumer) {
|
||||
if (result instanceof Function function) {
|
||||
result = function.apply(adaptedInboundArguments[i]);
|
||||
}
|
||||
else {
|
||||
((Consumer<Object>) result).accept(adaptedInboundArguments[i]);
|
||||
result = null;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private void handleSingleKStreamOutbound(Map<String, ResolvableType> resolvableTypes, ResolvableType outboundResolvableType,
|
||||
KStream<Object, Object> result, Iterator<String> outboundDefinitionIterator) {
|
||||
if (outboundDefinitionIterator.hasNext()) {
|
||||
String outbound = outboundDefinitionIterator.next();
|
||||
Object targetBean = handleSingleKStreamOutbound(result, outbound);
|
||||
kafkaStreamsBindingInformationCatalogue.addOutboundKStreamResolvable(targetBean,
|
||||
outboundResolvableType);
|
||||
|
||||
final String next = resolvableTypes.keySet().iterator().next();
|
||||
final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsBindingInformationCatalogue
|
||||
.getStreamsBuilderFactoryBeanPerBinding().get(next);
|
||||
this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding(outbound, streamsBuilderFactoryBean);
|
||||
}
|
||||
}
|
||||
|
||||
private Object handleSingleKStreamOutbound(KStream<Object, Object> result, String next) {
|
||||
Object targetBean = this.applicationContext.getBean(next);
|
||||
KStreamBoundElementFactory.KStreamWrapper
|
||||
boundElement = (KStreamBoundElementFactory.KStreamWrapper) targetBean;
|
||||
boundElement.wrap(result);
|
||||
return targetBean;
|
||||
}
|
||||
|
||||
@SuppressWarnings({ "unchecked", "rawtypes" })
|
||||
private void handleKStreamArrayOutbound(ResolvableType resolvableType, String functionName,
|
||||
KafkaStreamsBindableProxyFactory kafkaStreamsBindableProxyFactory,
|
||||
ResolvableType outboundResolvableType, Object[] result,
|
||||
StreamsBuilderFactoryBean streamsBuilderFactoryBean) {
|
||||
// Binding target as the output bindings were deferred in the KafkaStreamsBindableProxyFactory
|
||||
// due to the fact that it didn't know the returned array size. At this point in the execution,
|
||||
// we know exactly the number of outbound components (from the array length), so do the binding.
|
||||
final int length = result.length;
|
||||
|
||||
List<String> outputBindings = getOutputBindings(functionName, length);
|
||||
Iterator<String> iterator = outputBindings.iterator();
|
||||
BeanDefinitionRegistry registry = (BeanDefinitionRegistry) beanFactory;
|
||||
|
||||
for (Object o : result) {
|
||||
String next = iterator.next();
|
||||
kafkaStreamsBindableProxyFactory.addOutputBinding(next, KStream.class);
|
||||
RootBeanDefinition rootBeanDefinition1 = new RootBeanDefinition();
|
||||
rootBeanDefinition1.setInstanceSupplier(() -> kafkaStreamsBindableProxyFactory.getOutputHolders().get(next).getBoundTarget());
|
||||
registry.registerBeanDefinition(next, rootBeanDefinition1);
|
||||
|
||||
Object targetBean = this.applicationContext.getBean(next);
|
||||
|
||||
KStreamBoundElementFactory.KStreamWrapper
|
||||
boundElement = (KStreamBoundElementFactory.KStreamWrapper) targetBean;
|
||||
boundElement.wrap((KStream) o);
|
||||
|
||||
kafkaStreamsBindingInformationCatalogue.addOutboundKStreamResolvable(
|
||||
targetBean, outboundResolvableType != null ? outboundResolvableType : resolvableType.getGeneric(1));
|
||||
|
||||
this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding(next, streamsBuilderFactoryBean);
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> getOutputBindings(String functionName, int outputs) {
|
||||
List<String> outputBindings = this.streamFunctionProperties.getOutputBindings(functionName);
|
||||
List<String> outputBindingNames = new ArrayList<>();
|
||||
if (!CollectionUtils.isEmpty(outputBindings)) {
|
||||
outputBindingNames.addAll(outputBindings);
|
||||
return outputBindingNames;
|
||||
}
|
||||
else {
|
||||
for (int i = 0; i < outputs; i++) {
|
||||
outputBindingNames.add(String.format("%s-%s-%d", functionName, FunctionConstants.DEFAULT_OUTPUT_SUFFIX, i));
|
||||
}
|
||||
}
|
||||
return outputBindingNames;
|
||||
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked"})
|
||||
private Object[] adaptAndRetrieveInboundArguments(Map<String, ResolvableType> stringResolvableTypeMap,
|
||||
String functionName) {
|
||||
Object[] arguments = new Object[stringResolvableTypeMap.size()];
|
||||
int i = 0;
|
||||
for (String input : stringResolvableTypeMap.keySet()) {
|
||||
Class<?> parameterType = stringResolvableTypeMap.get(input).getRawClass();
|
||||
|
||||
if (input != null) {
|
||||
Object targetBean = applicationContext.getBean(input);
|
||||
BindingProperties bindingProperties = this.bindingServiceProperties.getBindingProperties(input);
|
||||
//Retrieve the StreamsConfig created for this method if available.
|
||||
//Otherwise, create the StreamsBuilderFactory and get the underlying config.
|
||||
if (!this.methodStreamsBuilderFactoryBeanMap.containsKey(functionName)) {
|
||||
StreamsBuilderFactoryBean streamsBuilderFactoryBean = buildStreamsBuilderAndRetrieveConfig(functionName, applicationContext,
|
||||
input, kafkaStreamsBinderConfigurationProperties, customizer, this.environment, bindingProperties);
|
||||
this.methodStreamsBuilderFactoryBeanMap.put(functionName, streamsBuilderFactoryBean);
|
||||
}
|
||||
try {
|
||||
StreamsBuilderFactoryBean streamsBuilderFactoryBean =
|
||||
this.methodStreamsBuilderFactoryBeanMap.get(functionName);
|
||||
StreamsBuilder streamsBuilder = streamsBuilderFactoryBean.getObject();
|
||||
final String applicationId = streamsBuilderFactoryBean.getStreamsConfiguration().getProperty(StreamsConfig.APPLICATION_ID_CONFIG);
|
||||
KafkaStreamsConsumerProperties extendedConsumerProperties =
|
||||
this.kafkaStreamsExtendedBindingProperties.getExtendedConsumerProperties(input);
|
||||
extendedConsumerProperties.setApplicationId(applicationId);
|
||||
//get state store spec
|
||||
|
||||
Serde<?> keySerde = this.keyValueSerdeResolver.getInboundKeySerde(extendedConsumerProperties, stringResolvableTypeMap.get(input));
|
||||
LOG.info("Key Serde used for " + input + ": " + keySerde.getClass().getName());
|
||||
Serde<?> valueSerde = bindingServiceProperties.getConsumerProperties(input).isUseNativeDecoding() ?
|
||||
getValueSerde(input, extendedConsumerProperties, stringResolvableTypeMap.get(input)) : Serdes.ByteArray();
|
||||
LOG.info("Value Serde used for " + input + ": " + valueSerde.getClass().getName());
|
||||
final Topology.AutoOffsetReset autoOffsetReset = getAutoOffsetReset(input, extendedConsumerProperties);
|
||||
|
||||
if (parameterType.isAssignableFrom(KStream.class)) {
|
||||
KStream<?, ?> stream = getKStream(input, bindingProperties, extendedConsumerProperties,
|
||||
streamsBuilder, keySerde, valueSerde, autoOffsetReset, i == 0);
|
||||
KStreamBoundElementFactory.KStreamWrapper kStreamWrapper =
|
||||
(KStreamBoundElementFactory.KStreamWrapper) targetBean;
|
||||
//wrap the proxy created during the initial target type binding with real object (KStream)
|
||||
kStreamWrapper.wrap((KStream<Object, Object>) stream);
|
||||
|
||||
this.kafkaStreamsBindingInformationCatalogue.addKeySerde((KStream<?, ?>) kStreamWrapper, keySerde);
|
||||
|
||||
this.kafkaStreamsBindingInformationCatalogue.addStreamBuilderFactoryPerBinding(input, streamsBuilderFactoryBean);
|
||||
this.kafkaStreamsBindingInformationCatalogue.addConsumerPropertiesPerSbfb(streamsBuilderFactoryBean,
|
||||
bindingServiceProperties.getConsumerProperties(input));
|
||||
|
||||
if (KStream.class.isAssignableFrom(stringResolvableTypeMap.get(input).getRawClass())) {
|
||||
final Class<?> valueClass =
|
||||
(stringResolvableTypeMap.get(input).getGeneric(1).getRawClass() != null)
|
||||
? (stringResolvableTypeMap.get(input).getGeneric(1).getRawClass()) : Object.class;
|
||||
if (this.kafkaStreamsBindingInformationCatalogue.isUseNativeDecoding(
|
||||
(KStream<?, ?>) kStreamWrapper)) {
|
||||
arguments[i] = stream;
|
||||
}
|
||||
else {
|
||||
arguments[i] = this.kafkaStreamsMessageConversionDelegate.deserializeOnInbound(
|
||||
valueClass, stream);
|
||||
}
|
||||
}
|
||||
|
||||
if (arguments[i] == null) {
|
||||
arguments[i] = stream;
|
||||
}
|
||||
Assert.notNull(arguments[i], "Problems encountered while adapting the function argument.");
|
||||
}
|
||||
else {
|
||||
handleKTableGlobalKTableInputs(arguments, i, input, parameterType, targetBean, streamsBuilderFactoryBean,
|
||||
streamsBuilder, extendedConsumerProperties, keySerde, valueSerde, autoOffsetReset, i == 0);
|
||||
}
|
||||
i++;
|
||||
}
|
||||
catch (Exception ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
return arguments;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setBeanFactory(BeanFactory beanFactory) throws BeansException {
|
||||
this.beanFactory = beanFactory;
|
||||
}
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
/*
|
||||
* Copyright 2019-2021 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.JaasLoginModuleConfiguration;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.kafka.security.jaas.KafkaJaasLoginModuleInitializer;
|
||||
|
||||
/**
|
||||
* Jaas configuration bean for Kafka Streams binder types.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @since 3.1.4
|
||||
*/
|
||||
@Configuration
|
||||
public class KafkaStreamsJaasConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean(KafkaJaasLoginModuleInitializer.class)
|
||||
public KafkaJaasLoginModuleInitializer jaasInitializer(
|
||||
KafkaBinderConfigurationProperties configurationProperties)
|
||||
throws IOException {
|
||||
KafkaJaasLoginModuleInitializer kafkaJaasLoginModuleInitializer = new KafkaJaasLoginModuleInitializer();
|
||||
JaasLoginModuleConfiguration jaas = configurationProperties.getJaas();
|
||||
if (jaas != null) {
|
||||
kafkaJaasLoginModuleInitializer.setLoginModule(jaas.getLoginModule());
|
||||
|
||||
KafkaJaasLoginModuleInitializer.ControlFlag controlFlag = jaas
|
||||
.getControlFlag();
|
||||
|
||||
if (controlFlag != null) {
|
||||
kafkaJaasLoginModuleInitializer.setControlFlag(controlFlag);
|
||||
}
|
||||
kafkaJaasLoginModuleInitializer.setOptions(jaas.getOptions());
|
||||
}
|
||||
return kafkaJaasLoginModuleInitializer;
|
||||
}
|
||||
}
|
||||
@@ -1,345 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
import org.apache.kafka.common.header.Header;
|
||||
import org.apache.kafka.common.header.Headers;
|
||||
import org.apache.kafka.common.header.internals.RecordHeader;
|
||||
import org.apache.kafka.common.serialization.Serde;
|
||||
import org.apache.kafka.common.serialization.Serializer;
|
||||
import org.apache.kafka.streams.KeyValue;
|
||||
import org.apache.kafka.streams.kstream.KStream;
|
||||
import org.apache.kafka.streams.processor.Processor;
|
||||
import org.apache.kafka.streams.processor.ProcessorContext;
|
||||
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.MessageHeaders;
|
||||
import org.springframework.messaging.converter.CompositeMessageConverter;
|
||||
import org.springframework.messaging.converter.MessageConverter;
|
||||
import org.springframework.messaging.support.MessageBuilder;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* Delegate for handling all framework level message conversions inbound and outbound on
|
||||
* {@link KStream}. If native encoding is not enabled, then serialization will be
|
||||
* performed on outbound messages based on a contentType. Similarly, if native decoding is
|
||||
* not enabled, deserialization will be performed on inbound messages based on a
|
||||
* contentType. Based on the contentType, a {@link MessageConverter} will be resolved.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @author Byungjun You
|
||||
*/
|
||||
public class KafkaStreamsMessageConversionDelegate {
|
||||
|
||||
private static final Log LOG = LogFactory
|
||||
.getLog(KafkaStreamsMessageConversionDelegate.class);
|
||||
|
||||
private static final ThreadLocal<KeyValue<Object, Object>> keyValueThreadLocal = new ThreadLocal<>();
|
||||
|
||||
private final CompositeMessageConverter compositeMessageConverter;
|
||||
|
||||
private final SendToDlqAndContinue sendToDlqAndContinue;
|
||||
|
||||
private final KafkaStreamsBindingInformationCatalogue kstreamBindingInformationCatalogue;
|
||||
|
||||
private final KafkaStreamsBinderConfigurationProperties kstreamBinderConfigurationProperties;
|
||||
|
||||
Exception[] failedWithDeserException = new Exception[1];
|
||||
|
||||
KafkaStreamsMessageConversionDelegate(
|
||||
CompositeMessageConverter compositeMessageConverter,
|
||||
SendToDlqAndContinue sendToDlqAndContinue,
|
||||
KafkaStreamsBindingInformationCatalogue kstreamBindingInformationCatalogue,
|
||||
KafkaStreamsBinderConfigurationProperties kstreamBinderConfigurationProperties) {
|
||||
this.compositeMessageConverter = compositeMessageConverter;
|
||||
this.sendToDlqAndContinue = sendToDlqAndContinue;
|
||||
this.kstreamBindingInformationCatalogue = kstreamBindingInformationCatalogue;
|
||||
this.kstreamBinderConfigurationProperties = kstreamBinderConfigurationProperties;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize {@link KStream} records on outbound based on contentType.
|
||||
* @param outboundBindTarget outbound KStream target
|
||||
* @return serialized KStream
|
||||
*/
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
public KStream serializeOnOutbound(KStream<?, ?> outboundBindTarget) {
|
||||
String contentType = this.kstreamBindingInformationCatalogue
|
||||
.getContentType(outboundBindTarget);
|
||||
MessageConverter messageConverter = this.compositeMessageConverter;
|
||||
final PerRecordContentTypeHolder perRecordContentTypeHolder = new PerRecordContentTypeHolder();
|
||||
|
||||
final KStream<?, ?> kStreamWithEnrichedHeaders = outboundBindTarget
|
||||
.filter((k, v) -> v != null)
|
||||
.mapValues((v) -> {
|
||||
Message<?> message = v instanceof Message<?> m ? m
|
||||
: MessageBuilder.withPayload(v).build();
|
||||
Map<String, Object> headers = new HashMap<>(message.getHeaders());
|
||||
if (StringUtils.hasText(contentType)) {
|
||||
headers.put(MessageHeaders.CONTENT_TYPE, contentType);
|
||||
}
|
||||
MessageHeaders messageHeaders = new MessageHeaders(headers);
|
||||
final Message<?> convertedMessage = messageConverter.toMessage(message.getPayload(), messageHeaders);
|
||||
perRecordContentTypeHolder.setContentType((String) messageHeaders.get(MessageHeaders.CONTENT_TYPE));
|
||||
return convertedMessage.getPayload();
|
||||
});
|
||||
|
||||
kStreamWithEnrichedHeaders.process(() -> new Processor() {
|
||||
|
||||
ProcessorContext context;
|
||||
|
||||
@Override
|
||||
public void init(ProcessorContext context) {
|
||||
this.context = context;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object key, Object value) {
|
||||
if (perRecordContentTypeHolder.contentType != null) {
|
||||
this.context.headers().remove(MessageHeaders.CONTENT_TYPE);
|
||||
final Header header;
|
||||
try {
|
||||
header = new RecordHeader(MessageHeaders.CONTENT_TYPE,
|
||||
new ObjectMapper().writeValueAsBytes(perRecordContentTypeHolder.contentType));
|
||||
this.context.headers().add(header);
|
||||
}
|
||||
catch (Exception e) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Could not add content type header");
|
||||
}
|
||||
}
|
||||
perRecordContentTypeHolder.unsetContentType();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
|
||||
}
|
||||
});
|
||||
|
||||
return kStreamWithEnrichedHeaders;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deserialize incoming {@link KStream} based on content type.
|
||||
* @param valueClass on KStream value
|
||||
* @param bindingTarget inbound KStream target
|
||||
* @return deserialized KStream
|
||||
*/
|
||||
@SuppressWarnings({ "unchecked", "rawtypes" })
|
||||
public KStream deserializeOnInbound(Class<?> valueClass,
|
||||
KStream<?, ?> bindingTarget) {
|
||||
MessageConverter messageConverter = this.compositeMessageConverter;
|
||||
final PerRecordContentTypeHolder perRecordContentTypeHolder = new PerRecordContentTypeHolder();
|
||||
|
||||
resolvePerRecordContentType(bindingTarget, perRecordContentTypeHolder);
|
||||
|
||||
// Deserialize using a branching strategy
|
||||
KStream<?, ?>[] branch = bindingTarget.branch(
|
||||
// First filter where the message is converted and return true if
|
||||
// everything went well, return false otherwise.
|
||||
(o, o2) -> {
|
||||
boolean isValidRecord = false;
|
||||
|
||||
try {
|
||||
// if the record is a tombstone, ignore and exit from processing
|
||||
// further.
|
||||
if (o2 != null) {
|
||||
if (o2 instanceof Message || o2 instanceof String
|
||||
|| o2 instanceof byte[]) {
|
||||
Message<?> m1 = null;
|
||||
if (o2 instanceof Message message) {
|
||||
m1 = perRecordContentTypeHolder.contentType != null
|
||||
? MessageBuilder.fromMessage(message)
|
||||
.setHeader(
|
||||
MessageHeaders.CONTENT_TYPE,
|
||||
perRecordContentTypeHolder.contentType)
|
||||
.build()
|
||||
: message;
|
||||
}
|
||||
else {
|
||||
m1 = perRecordContentTypeHolder.contentType != null
|
||||
? MessageBuilder.withPayload(o2).setHeader(
|
||||
MessageHeaders.CONTENT_TYPE,
|
||||
perRecordContentTypeHolder.contentType)
|
||||
.build()
|
||||
: MessageBuilder.withPayload(o2).build();
|
||||
}
|
||||
convertAndSetMessage(o, valueClass, messageConverter, m1);
|
||||
}
|
||||
else {
|
||||
keyValueThreadLocal.set(new KeyValue<>(o, o2));
|
||||
}
|
||||
isValidRecord = true;
|
||||
}
|
||||
else {
|
||||
LOG.info(
|
||||
"Received a tombstone record. This will be skipped from further processing.");
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
LOG.warn(
|
||||
"Deserialization has failed. This will be skipped from further processing.",
|
||||
e);
|
||||
// pass through
|
||||
failedWithDeserException[0] = e;
|
||||
}
|
||||
return isValidRecord;
|
||||
},
|
||||
// second filter that catches any messages for which an exception thrown
|
||||
// in the first filter above.
|
||||
(k, v) -> true);
|
||||
// process errors from the second filter in the branch above.
|
||||
processErrorFromDeserialization(bindingTarget, branch[1], failedWithDeserException);
|
||||
|
||||
// first branch above is the branch where the messages are converted, let it go
|
||||
// through further processing.
|
||||
return branch[0].mapValues((o2) -> {
|
||||
Object objectValue = keyValueThreadLocal.get().value;
|
||||
keyValueThreadLocal.remove();
|
||||
return objectValue;
|
||||
});
|
||||
}
|
||||
|
||||
@SuppressWarnings({ "unchecked", "rawtypes" })
|
||||
private void resolvePerRecordContentType(KStream<?, ?> outboundBindTarget,
|
||||
PerRecordContentTypeHolder perRecordContentTypeHolder) {
|
||||
outboundBindTarget.process(() -> new Processor() {
|
||||
|
||||
ProcessorContext context;
|
||||
|
||||
@Override
|
||||
public void init(ProcessorContext context) {
|
||||
this.context = context;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object key, Object value) {
|
||||
final Headers headers = this.context.headers();
|
||||
final Iterable<Header> contentTypes = headers
|
||||
.headers(MessageHeaders.CONTENT_TYPE);
|
||||
if (contentTypes != null && contentTypes.iterator().hasNext()) {
|
||||
final String contentType = new String(
|
||||
contentTypes.iterator().next().value());
|
||||
// remove leading and trailing quotes
|
||||
final String cleanContentType = StringUtils.replace(contentType, "\"",
|
||||
"");
|
||||
perRecordContentTypeHolder.setContentType(cleanContentType);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void convertAndSetMessage(Object o, Class<?> valueClass,
|
||||
MessageConverter messageConverter, Message<?> msg) {
|
||||
Object result = valueClass.isAssignableFrom(msg.getPayload().getClass())
|
||||
? msg.getPayload() : messageConverter.fromMessage(msg, valueClass);
|
||||
|
||||
Assert.notNull(result, "Failed to convert message " + msg);
|
||||
|
||||
keyValueThreadLocal.set(new KeyValue<>(o, result));
|
||||
}
|
||||
|
||||
@SuppressWarnings({ "unchecked", "rawtypes" })
|
||||
private void processErrorFromDeserialization(KStream<?, ?> bindingTarget,
|
||||
KStream<?, ?> branch, Exception[] exception) {
|
||||
branch.process(() -> new Processor() {
|
||||
ProcessorContext context;
|
||||
|
||||
@Override
|
||||
public void init(ProcessorContext context) {
|
||||
this.context = context;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object o, Object o2) {
|
||||
// Only continue if the record was not a tombstone.
|
||||
if (o2 != null) {
|
||||
if (KafkaStreamsMessageConversionDelegate.this.kstreamBindingInformationCatalogue
|
||||
.isDlqEnabled(bindingTarget)) {
|
||||
if (o2 instanceof Message message) {
|
||||
|
||||
// We need to convert the key to a byte[] before sending to DLQ.
|
||||
Serde keySerde = kstreamBindingInformationCatalogue.getKeySerde(bindingTarget);
|
||||
Serializer keySerializer = keySerde.serializer();
|
||||
byte[] keyBytes = keySerializer.serialize(null, o);
|
||||
|
||||
ConsumerRecord consumerRecord = new ConsumerRecord(this.context.topic(), this.context.partition(), this.context.offset(),
|
||||
keyBytes, message.getPayload());
|
||||
|
||||
KafkaStreamsMessageConversionDelegate.this.sendToDlqAndContinue
|
||||
.sendToDlq(consumerRecord, exception[0]);
|
||||
}
|
||||
else {
|
||||
ConsumerRecord consumerRecord = new ConsumerRecord(this.context.topic(), this.context.partition(), this.context.offset(),
|
||||
o, o2);
|
||||
KafkaStreamsMessageConversionDelegate.this.sendToDlqAndContinue
|
||||
.sendToDlq(consumerRecord, exception[0]);
|
||||
}
|
||||
}
|
||||
else if (KafkaStreamsMessageConversionDelegate.this.kstreamBinderConfigurationProperties
|
||||
.getSerdeError() == KafkaStreamsBinderConfigurationProperties.SerdeError.logAndFail) {
|
||||
throw new IllegalStateException("Inbound deserialization failed. "
|
||||
+ "Stopping further processing of records.");
|
||||
}
|
||||
else if (KafkaStreamsMessageConversionDelegate.this.kstreamBinderConfigurationProperties
|
||||
.getSerdeError() == KafkaStreamsBinderConfigurationProperties.SerdeError.logAndContinue) {
|
||||
// quietly passing through. No action needed, this is similar to
|
||||
// log and continue.
|
||||
LOG.error(
|
||||
"Inbound deserialization failed. Skipping this record and continuing.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private static class PerRecordContentTypeHolder {
|
||||
|
||||
String contentType;
|
||||
|
||||
void setContentType(String contentType) {
|
||||
this.contentType = contentType;
|
||||
}
|
||||
|
||||
void unsetContentType() {
|
||||
this.contentType = null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2019 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import org.apache.kafka.streams.KafkaStreams;
|
||||
import org.apache.kafka.streams.StreamsConfig;
|
||||
|
||||
import org.springframework.kafka.config.StreamsBuilderFactoryBean;
|
||||
|
||||
/**
|
||||
* An internal registry for holding {@link KafkaStreams} objects maintained through
|
||||
* {@link StreamsBuilderFactoryManager}.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
public class KafkaStreamsRegistry {
|
||||
|
||||
private final Map<KafkaStreams, StreamsBuilderFactoryBean> streamsBuilderFactoryBeanMap = new ConcurrentHashMap<>();
|
||||
|
||||
private final Set<KafkaStreams> kafkaStreams = ConcurrentHashMap.newKeySet();
|
||||
|
||||
Set<KafkaStreams> getKafkaStreams() {
|
||||
Set<KafkaStreams> currentlyRunningKafkaStreams = new HashSet<>();
|
||||
for (KafkaStreams ks : this.kafkaStreams) {
|
||||
final StreamsBuilderFactoryBean streamsBuilderFactoryBean = streamsBuilderFactoryBeanMap.get(ks);
|
||||
if (streamsBuilderFactoryBean.isRunning()) {
|
||||
currentlyRunningKafkaStreams.add(ks);
|
||||
}
|
||||
}
|
||||
return currentlyRunningKafkaStreams;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register the {@link KafkaStreams} object created in the application.
|
||||
* @param streamsBuilderFactoryBean {@link StreamsBuilderFactoryBean}
|
||||
*/
|
||||
void registerKafkaStreams(StreamsBuilderFactoryBean streamsBuilderFactoryBean) {
|
||||
final KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams();
|
||||
this.kafkaStreams.add(kafkaStreams);
|
||||
this.streamsBuilderFactoryBeanMap.put(kafkaStreams, streamsBuilderFactoryBean);
|
||||
}
|
||||
|
||||
void unregisterKafkaStreams(KafkaStreams kafkaStreams) {
|
||||
this.kafkaStreams.remove(kafkaStreams);
|
||||
this.streamsBuilderFactoryBeanMap.remove(kafkaStreams);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param kafkaStreams {@link KafkaStreams} object
|
||||
* @return Corresponding {@link StreamsBuilderFactoryBean}.
|
||||
*/
|
||||
StreamsBuilderFactoryBean streamBuilderFactoryBean(KafkaStreams kafkaStreams) {
|
||||
return this.streamsBuilderFactoryBeanMap.get(kafkaStreams);
|
||||
}
|
||||
|
||||
public StreamsBuilderFactoryBean streamsBuilderFactoryBean(String applicationId) {
|
||||
final Optional<StreamsBuilderFactoryBean> first = this.streamsBuilderFactoryBeanMap.values()
|
||||
.stream()
|
||||
.filter(streamsBuilderFactoryBean -> streamsBuilderFactoryBean.isRunning() && streamsBuilderFactoryBean
|
||||
.getStreamsConfiguration().getProperty(StreamsConfig.APPLICATION_ID_CONFIG)
|
||||
.equals(applicationId))
|
||||
.findFirst();
|
||||
return first.orElse(null);
|
||||
}
|
||||
|
||||
public List<StreamsBuilderFactoryBean> streamsBuilderFactoryBeans() {
|
||||
return new ArrayList<>(this.streamsBuilderFactoryBeanMap.values());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.Collection;
|
||||
import java.util.StringJoiner;
|
||||
|
||||
import org.apache.kafka.streams.KafkaStreams;
|
||||
|
||||
import org.springframework.core.log.LogAccessor;
|
||||
import org.springframework.lang.Nullable;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.ReflectionUtils;
|
||||
|
||||
/**
|
||||
* A facade to access topology info for a Kafka Streams application in a version agnostic
|
||||
* manner.
|
||||
* <p>Before kafka-streams 3.1 the topology exists at 'KafkaStreams.internalTopologyBuilder'.
|
||||
* Starting in kafka-streams 3.1 the topology exists at 'KafkaStreams.topologyMetadata'.
|
||||
*
|
||||
* @author Chris Bono
|
||||
* @since 3.2.6
|
||||
*/
|
||||
class KafkaStreamsVersionAgnosticTopologyInfoFacade {
|
||||
|
||||
private final LogAccessor logger = new LogAccessor(KafkaStreamsVersionAgnosticTopologyInfoFacade.class);
|
||||
|
||||
@Nullable
|
||||
private Field topologyInfoField;
|
||||
|
||||
@Nullable
|
||||
private Method sourceTopicsForStoreMethod;
|
||||
|
||||
private boolean sourceTopicsForStoreMethodHasTwoArgs;
|
||||
|
||||
KafkaStreamsVersionAgnosticTopologyInfoFacade() {
|
||||
this(KafkaStreams.class);
|
||||
}
|
||||
|
||||
KafkaStreamsVersionAgnosticTopologyInfoFacade(Class<?> rootClass) {
|
||||
|
||||
// First look for KafkaStreams.internalTopologyBuilder (exists in kafka-streams <= 3.0)
|
||||
Field internalTopologyBuilderField = ReflectionUtils.findField(rootClass, "internalTopologyBuilder");
|
||||
if (internalTopologyBuilderField != null) {
|
||||
internalTopologyBuilderField.setAccessible(true);
|
||||
this.topologyInfoField = internalTopologyBuilderField;
|
||||
this.sourceTopicsForStoreMethod = ReflectionUtils.findMethod(internalTopologyBuilderField.getType(), "sourceTopicsForStore", String.class);
|
||||
}
|
||||
|
||||
// Otherwise look for KafkaStreams.topologyMetadata (exists in kafka-streams >= 3.1)
|
||||
if (this.sourceTopicsForStoreMethod == null) {
|
||||
Field topologyMetadataField = ReflectionUtils.findField(rootClass, "topologyMetadata");
|
||||
if (topologyMetadataField != null) {
|
||||
topologyMetadataField.setAccessible(true);
|
||||
this.topologyInfoField = topologyMetadataField;
|
||||
this.sourceTopicsForStoreMethod = ReflectionUtils.findMethod(topologyMetadataField.getType(), "sourceTopicsForStore", String.class);
|
||||
if (this.sourceTopicsForStoreMethod == null) {
|
||||
// The sourceTopicsForStore method has extra arg in kafka-streams >= 3.3
|
||||
this.sourceTopicsForStoreMethod = ReflectionUtils.findMethod(topologyMetadataField.getType(), "sourceTopicsForStore", String.class, String.class);
|
||||
this.sourceTopicsForStoreMethodHasTwoArgs = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (this.sourceTopicsForStoreMethod != null) {
|
||||
this.sourceTopicsForStoreMethod.setAccessible(true);
|
||||
logger.info(() -> "Using " + methodDescription(this.sourceTopicsForStoreMethod));
|
||||
}
|
||||
else {
|
||||
logger.warn("Could not find 'topologyMetadata.sourceTopicsForStore' or 'internalTopologyBuilder.sourceTopicsForStore' " +
|
||||
"from KafkaStreams class - will be unable to reason about state stores.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if a state store is actually available to a KafkaStreams instance by
|
||||
* querying the topology info source topics for the requested store.
|
||||
*
|
||||
* @param kafkaStreams the streams app
|
||||
* @param storeName the name of the state store
|
||||
* @return {@code true} if state store is available or {@code false} if the state store is
|
||||
* not available or there was a problem reflecting on the topology info
|
||||
*/
|
||||
boolean streamsAppActuallyHasStore(KafkaStreams kafkaStreams, String storeName) {
|
||||
if (this.sourceTopicsForStoreMethod == null) {
|
||||
logger.warn("Unable to reason about state store because sourceTopicsForStore method was not found - returning false");
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
Object topologyInfo = ReflectionUtils.getField(this.topologyInfoField, kafkaStreams);
|
||||
if (topologyInfo == null) {
|
||||
logger.warn("Unable to reason about state store because topologyInfo field was null - returning false");
|
||||
return false;
|
||||
}
|
||||
Object[] args = this.sourceTopicsForStoreMethodHasTwoArgs ?
|
||||
new Object[] { storeName, null } : new Object[] { storeName };
|
||||
Collection<String> sourceTopicsForStore = (Collection<String>)
|
||||
ReflectionUtils.invokeMethod(this.sourceTopicsForStoreMethod, topologyInfo, args);
|
||||
return !CollectionUtils.isEmpty(sourceTopicsForStore);
|
||||
}
|
||||
catch (Exception ex) {
|
||||
logger.error(ex, () -> "Unable to reason about state store due to error: " + ex.getMessage() + " - returning false");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private String methodDescription(Method method) {
|
||||
StringJoiner sj = new StringJoiner(",", method.getName() + "(", ")");
|
||||
for (Class<?> parameterType : method.getParameterTypes()) {
|
||||
sj.add(parameterType.getTypeName());
|
||||
}
|
||||
return "method " + method.getDeclaringClass().getTypeName() + '.' + sj.toString();
|
||||
}
|
||||
}
|
||||
@@ -1,333 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2021 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.kafka.common.serialization.Serde;
|
||||
import org.apache.kafka.common.serialization.Serdes;
|
||||
import org.apache.kafka.common.utils.Utils;
|
||||
import org.apache.kafka.streams.kstream.GlobalKTable;
|
||||
import org.apache.kafka.streams.kstream.KStream;
|
||||
import org.apache.kafka.streams.kstream.KTable;
|
||||
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.cloud.stream.binder.ConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.ProducerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsProducerProperties;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ApplicationContextAware;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.core.ResolvableType;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* Resolver for key and value Serde.
|
||||
*
|
||||
* On the inbound, if native decoding is enabled, then any deserialization on the value is
|
||||
* handled by Kafka. First, we look for any key/value Serde set on the binding itself, if
|
||||
* that is not available then look at the common Serde set at the global level. If that
|
||||
* fails, it falls back to byte[]. If native decoding is disabled, then the binder will do
|
||||
* the deserialization on value and ignore any Serde set for value and rely on the
|
||||
* contentType provided. Keys are always deserialized at the broker.
|
||||
*
|
||||
*
|
||||
* Same rules apply on the outbound. If native encoding is enabled, then value
|
||||
* serialization is done at the broker using any binder level Serde for value, if not
|
||||
* using common Serde, if not, then byte[]. If native encoding is disabled, then the
|
||||
* binder will do serialization using a contentType. Keys are always serialized by the
|
||||
* broker.
|
||||
*
|
||||
* For state store, use serdes class specified in
|
||||
* {@link org.springframework.cloud.stream.binder.kafka.streams.annotations.KafkaStreamsStateStore}
|
||||
* to create Serde accordingly.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @author Lei Chen
|
||||
* @author Eduard Domínguez
|
||||
* @author Chris Bono
|
||||
*/
|
||||
public class KeyValueSerdeResolver implements ApplicationContextAware {
|
||||
|
||||
private final Map<String, Object> streamConfigGlobalProperties;
|
||||
|
||||
private final KafkaStreamsBinderConfigurationProperties binderConfigurationProperties;
|
||||
|
||||
private ConfigurableApplicationContext context;
|
||||
|
||||
KeyValueSerdeResolver(Map<String, Object> streamConfigGlobalProperties,
|
||||
KafkaStreamsBinderConfigurationProperties binderConfigurationProperties) {
|
||||
this.streamConfigGlobalProperties = streamConfigGlobalProperties;
|
||||
this.binderConfigurationProperties = binderConfigurationProperties;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provide the {@link Serde} for inbound key.
|
||||
* @param extendedConsumerProperties binding level extended
|
||||
* {@link KafkaStreamsConsumerProperties}
|
||||
* @return configurd {@link Serde} for the inbound key.
|
||||
*/
|
||||
public Serde<?> getInboundKeySerde(
|
||||
KafkaStreamsConsumerProperties extendedConsumerProperties) {
|
||||
String keySerdeString = extendedConsumerProperties.getKeySerde();
|
||||
|
||||
return getKeySerde(keySerdeString, extendedConsumerProperties.getConfiguration());
|
||||
}
|
||||
|
||||
public Serde<?> getInboundKeySerde(
|
||||
KafkaStreamsConsumerProperties extendedConsumerProperties, ResolvableType resolvableType) {
|
||||
String keySerdeString = extendedConsumerProperties.getKeySerde();
|
||||
|
||||
return getKeySerde(keySerdeString, resolvableType, extendedConsumerProperties.getConfiguration());
|
||||
}
|
||||
|
||||
/**
|
||||
* Provide the {@link Serde} for inbound value.
|
||||
* @param consumerProperties {@link ConsumerProperties} on binding
|
||||
* @param extendedConsumerProperties binding level extended
|
||||
* {@link KafkaStreamsConsumerProperties}
|
||||
* @return configurd {@link Serde} for the inbound value.
|
||||
*/
|
||||
public Serde<?> getInboundValueSerde(ConsumerProperties consumerProperties,
|
||||
KafkaStreamsConsumerProperties extendedConsumerProperties) {
|
||||
Serde<?> valueSerde;
|
||||
|
||||
String valueSerdeString = extendedConsumerProperties.getValueSerde();
|
||||
try {
|
||||
if (consumerProperties != null && consumerProperties.isUseNativeDecoding()) {
|
||||
valueSerde = getValueSerde(valueSerdeString, extendedConsumerProperties.getConfiguration());
|
||||
}
|
||||
else {
|
||||
valueSerde = Serdes.ByteArray();
|
||||
}
|
||||
}
|
||||
catch (ClassNotFoundException ex) {
|
||||
throw new IllegalStateException("Serde class not found: ", ex);
|
||||
}
|
||||
return valueSerde;
|
||||
}
|
||||
|
||||
public Serde<?> getInboundValueSerde(ConsumerProperties consumerProperties,
|
||||
KafkaStreamsConsumerProperties extendedConsumerProperties,
|
||||
ResolvableType resolvableType) {
|
||||
Serde<?> valueSerde;
|
||||
|
||||
String valueSerdeString = extendedConsumerProperties.getValueSerde();
|
||||
try {
|
||||
if (consumerProperties != null && consumerProperties.isUseNativeDecoding()) {
|
||||
valueSerde = getValueSerde(valueSerdeString, resolvableType, extendedConsumerProperties.getConfiguration());
|
||||
}
|
||||
else {
|
||||
valueSerde = Serdes.ByteArray();
|
||||
}
|
||||
}
|
||||
catch (ClassNotFoundException ex) {
|
||||
throw new IllegalStateException("Serde class not found: ", ex);
|
||||
}
|
||||
return valueSerde;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provide the {@link Serde} for outbound key.
|
||||
* @param properties binding level extended {@link KafkaStreamsProducerProperties}
|
||||
* @return configurd {@link Serde} for the outbound key.
|
||||
*/
|
||||
public Serde<?> getOuboundKeySerde(KafkaStreamsProducerProperties properties) {
|
||||
return getKeySerde(properties.getKeySerde(), properties.getConfiguration());
|
||||
}
|
||||
|
||||
public Serde<?> getOuboundKeySerde(KafkaStreamsProducerProperties properties, ResolvableType resolvableType) {
|
||||
return getKeySerde(properties.getKeySerde(), resolvableType, properties.getConfiguration());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Provide the {@link Serde} for outbound value.
|
||||
* @param producerProperties {@link ProducerProperties} on binding
|
||||
* @param kafkaStreamsProducerProperties binding level extended
|
||||
* {@link KafkaStreamsProducerProperties}
|
||||
* @return configurd {@link Serde} for the outbound value.
|
||||
*/
|
||||
public Serde<?> getOutboundValueSerde(ProducerProperties producerProperties,
|
||||
KafkaStreamsProducerProperties kafkaStreamsProducerProperties) {
|
||||
Serde<?> valueSerde;
|
||||
try {
|
||||
if (producerProperties.isUseNativeEncoding()) {
|
||||
valueSerde = getValueSerde(
|
||||
kafkaStreamsProducerProperties.getValueSerde(), kafkaStreamsProducerProperties.getConfiguration());
|
||||
}
|
||||
else {
|
||||
valueSerde = Serdes.ByteArray();
|
||||
}
|
||||
}
|
||||
catch (ClassNotFoundException ex) {
|
||||
throw new IllegalStateException("Serde class not found: ", ex);
|
||||
}
|
||||
return valueSerde;
|
||||
}
|
||||
|
||||
public Serde<?> getOutboundValueSerde(ProducerProperties producerProperties,
|
||||
KafkaStreamsProducerProperties kafkaStreamsProducerProperties, ResolvableType resolvableType) {
|
||||
Serde<?> valueSerde;
|
||||
try {
|
||||
if (producerProperties.isUseNativeEncoding()) {
|
||||
valueSerde = getValueSerde(
|
||||
kafkaStreamsProducerProperties.getValueSerde(), resolvableType, kafkaStreamsProducerProperties.getConfiguration());
|
||||
}
|
||||
else {
|
||||
valueSerde = Serdes.ByteArray();
|
||||
}
|
||||
}
|
||||
catch (ClassNotFoundException ex) {
|
||||
throw new IllegalStateException("Serde class not found: ", ex);
|
||||
}
|
||||
return valueSerde;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provide the {@link Serde} for state store.
|
||||
* @param keySerdeString serde class used for key
|
||||
* @return {@link Serde} for the state store key.
|
||||
*/
|
||||
public Serde<?> getStateStoreKeySerde(String keySerdeString) {
|
||||
return getKeySerde(keySerdeString, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Provide the {@link Serde} for state store value.
|
||||
* @param valueSerdeString serde class used for value
|
||||
* @return {@link Serde} for the state store value.
|
||||
*/
|
||||
public Serde<?> getStateStoreValueSerde(String valueSerdeString) {
|
||||
try {
|
||||
return getValueSerde(valueSerdeString, null);
|
||||
}
|
||||
catch (ClassNotFoundException ex) {
|
||||
throw new IllegalStateException("Serde class not found: ", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private Serde<?> getKeySerde(String keySerdeString, Map<String, ?> extendedConfiguration) {
|
||||
Serde<?> keySerde;
|
||||
try {
|
||||
if (StringUtils.hasText(keySerdeString)) {
|
||||
keySerde = Utils.newInstance(keySerdeString, Serde.class);
|
||||
}
|
||||
else {
|
||||
keySerde = getFallbackSerde("default.key.serde");
|
||||
}
|
||||
keySerde.configure(combineStreamConfigProperties(extendedConfiguration), true);
|
||||
}
|
||||
catch (ClassNotFoundException ex) {
|
||||
throw new IllegalStateException("Serde class not found: ", ex);
|
||||
}
|
||||
return keySerde;
|
||||
}
|
||||
|
||||
private Serde<?> getKeySerde(String keySerdeString, ResolvableType resolvableType, Map<String, ?> extendedConfiguration) {
|
||||
Serde<?> keySerde;
|
||||
try {
|
||||
if (StringUtils.hasText(keySerdeString)) {
|
||||
keySerde = Utils.newInstance(keySerdeString, Serde.class);
|
||||
}
|
||||
else {
|
||||
keySerde = Serdes.ByteArray();
|
||||
if (resolvableType != null &&
|
||||
(isResolvalbeKafkaStreamsType(resolvableType) || isResolvableKStreamArrayType(resolvableType))) {
|
||||
ResolvableType targetType = resolvableType.isArray() ? resolvableType.getComponentType().getGeneric(0) : resolvableType.getGeneric(0);
|
||||
Serde<?> fallbackSerde = getFallbackSerde("default.key.serde");
|
||||
keySerde = SerdeResolverUtils.resolveForType(this.context, targetType, fallbackSerde);
|
||||
}
|
||||
}
|
||||
keySerde.configure(combineStreamConfigProperties(extendedConfiguration), true);
|
||||
}
|
||||
catch (ClassNotFoundException ex) {
|
||||
throw new IllegalStateException("Serde class not found: ", ex);
|
||||
}
|
||||
return keySerde;
|
||||
}
|
||||
|
||||
private boolean isResolvableKStreamArrayType(ResolvableType resolvableType) {
|
||||
return resolvableType.isArray() &&
|
||||
KStream.class.isAssignableFrom(resolvableType.getComponentType().getRawClass());
|
||||
}
|
||||
|
||||
private boolean isResolvalbeKafkaStreamsType(ResolvableType resolvableType) {
|
||||
return resolvableType.getRawClass() != null && (KStream.class.isAssignableFrom(resolvableType.getRawClass()) || KTable.class.isAssignableFrom(resolvableType.getRawClass()) ||
|
||||
GlobalKTable.class.isAssignableFrom(resolvableType.getRawClass()));
|
||||
}
|
||||
|
||||
private Serde<?> getValueSerde(String valueSerdeString, Map<String, ?> extendedConfiguration)
|
||||
throws ClassNotFoundException {
|
||||
Serde<?> valueSerde;
|
||||
if (StringUtils.hasText(valueSerdeString)) {
|
||||
valueSerde = Utils.newInstance(valueSerdeString, Serde.class);
|
||||
}
|
||||
else {
|
||||
valueSerde = getFallbackSerde("default.value.serde");
|
||||
}
|
||||
valueSerde.configure(combineStreamConfigProperties(extendedConfiguration), false);
|
||||
return valueSerde;
|
||||
}
|
||||
|
||||
private Serde<?> getFallbackSerde(String s) throws ClassNotFoundException {
|
||||
return this.binderConfigurationProperties.getConfiguration()
|
||||
.containsKey(s)
|
||||
? Utils.newInstance(this.binderConfigurationProperties
|
||||
.getConfiguration().get(s),
|
||||
Serde.class)
|
||||
: Serdes.ByteArray();
|
||||
}
|
||||
|
||||
private Serde<?> getValueSerde(String valueSerdeString, ResolvableType resolvableType, Map<String, ?> extendedConfiguration)
|
||||
throws ClassNotFoundException {
|
||||
Serde<?> valueSerde;
|
||||
if (StringUtils.hasText(valueSerdeString)) {
|
||||
valueSerde = Utils.newInstance(valueSerdeString, Serde.class);
|
||||
}
|
||||
else {
|
||||
valueSerde = Serdes.ByteArray();
|
||||
if (resolvableType != null && ((isResolvalbeKafkaStreamsType(resolvableType)) ||
|
||||
(isResolvableKStreamArrayType(resolvableType)))) {
|
||||
ResolvableType targetType = resolvableType.isArray() ? resolvableType.getComponentType().getGeneric(1) : resolvableType.getGeneric(1);
|
||||
Serde<?> fallbackSerde = getFallbackSerde("default.value.serde");
|
||||
valueSerde = SerdeResolverUtils.resolveForType(this.context, targetType, fallbackSerde);
|
||||
}
|
||||
}
|
||||
valueSerde.configure(combineStreamConfigProperties(extendedConfiguration), false);
|
||||
return valueSerde;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
|
||||
context = (ConfigurableApplicationContext) applicationContext;
|
||||
}
|
||||
|
||||
private Map<String, ?> combineStreamConfigProperties(Map<String, ?> extendedConfiguration) {
|
||||
if (extendedConfiguration != null && !extendedConfiguration.isEmpty()) {
|
||||
Map<String, Object> streamConfiguration = new HashMap<>(this.streamConfigGlobalProperties);
|
||||
streamConfiguration.putAll(extendedConfiguration);
|
||||
return streamConfiguration;
|
||||
}
|
||||
else {
|
||||
return this.streamConfigGlobalProperties;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
/*
|
||||
* Copyright 2019-2021 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
|
||||
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
|
||||
import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* @author Soby Chacko
|
||||
* @since 3.0.2
|
||||
*/
|
||||
@Configuration
|
||||
public class MultiBinderPropertiesConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConfigurationProperties(prefix = "spring.cloud.stream.kafka.streams.binder")
|
||||
@ConditionalOnBean(name = "outerContext")
|
||||
public KafkaBinderConfigurationProperties binderConfigurationProperties(KafkaProperties kafkaProperties) {
|
||||
return new KafkaStreamsBinderConfigurationProperties(kafkaProperties);
|
||||
}
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
/*
|
||||
* Copyright 2018-2019 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
|
||||
import org.springframework.kafka.listener.ConsumerRecordRecoverer;
|
||||
import org.springframework.kafka.listener.DeadLetterPublishingRecoverer;
|
||||
|
||||
/**
|
||||
* Custom implementation for {@link ConsumerRecordRecoverer} that keeps a collection of
|
||||
* recoverer objects per input topics. These topics might be per input binding or multiplexed
|
||||
* topics in a single binding.
|
||||
*
|
||||
* @author Soby Chacko
|
||||
* @since 2.0.0
|
||||
*/
|
||||
public class SendToDlqAndContinue implements ConsumerRecordRecoverer {
|
||||
|
||||
/**
|
||||
* DLQ dispatcher per topic in the application context. The key here is not the actual
|
||||
* DLQ topic but the incoming topic that caused the error.
|
||||
*/
|
||||
private Map<String, DeadLetterPublishingRecoverer> dlqDispatchers = new HashMap<>();
|
||||
|
||||
/**
|
||||
* For a given topic, send the key/value record to DLQ topic.
|
||||
*
|
||||
* @param consumerRecord consumer record
|
||||
* @param exception exception
|
||||
*/
|
||||
public void sendToDlq(ConsumerRecord<?, ?> consumerRecord, Exception exception) {
|
||||
DeadLetterPublishingRecoverer kafkaStreamsDlqDispatch = this.dlqDispatchers.get(consumerRecord.topic());
|
||||
kafkaStreamsDlqDispatch.accept(consumerRecord, exception);
|
||||
}
|
||||
|
||||
void addKStreamDlqDispatch(String topic,
|
||||
DeadLetterPublishingRecoverer kafkaStreamsDlqDispatch) {
|
||||
this.dlqDispatchers.put(topic, kafkaStreamsDlqDispatch);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void accept(ConsumerRecord<?, ?> consumerRecord, Exception e) {
|
||||
this.dlqDispatchers.get(consumerRecord.topic()).accept(consumerRecord, e);
|
||||
}
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022-2022 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.lang.reflect.Type;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.kafka.common.serialization.Serde;
|
||||
import org.apache.kafka.common.serialization.Serdes;
|
||||
|
||||
import org.springframework.beans.factory.BeanFactoryUtils;
|
||||
import org.springframework.beans.factory.config.BeanDefinition;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.core.ResolvableType;
|
||||
import org.springframework.kafka.support.serializer.JsonSerde;
|
||||
import org.springframework.lang.Nullable;
|
||||
|
||||
/**
|
||||
* Utility class that contains various methods to help resolve {@link Serde Serdes}.
|
||||
*
|
||||
* @author Chris Bono
|
||||
* @since 4.0
|
||||
*/
|
||||
abstract class SerdeResolverUtils {
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(SerdeResolverUtils.class);
|
||||
|
||||
/** Classnames of the standard built-in Serdes supported in {@link Serdes#serdeFrom(Class)}. */
|
||||
private static final Set<String> STANDARD_SERDE_CLASSNAMES = Set.of(
|
||||
Serdes.String().getClass().getName(),
|
||||
Serdes.Short().getClass().getName(),
|
||||
Serdes.Integer().getClass().getName(),
|
||||
Serdes.Long().getClass().getName(),
|
||||
Serdes.Float().getClass().getName(),
|
||||
Serdes.Double().getClass().getName(),
|
||||
Serdes.ByteArray().getClass().getName(),
|
||||
Serdes.ByteBuffer().getClass().getName(),
|
||||
Serdes.Bytes().getClass().getName(),
|
||||
Serdes.UUID().getClass().getName());
|
||||
|
||||
/**
|
||||
* Return the {@code Serde<?>} to use for the specified type using the following steps until a match is found.
|
||||
* <p><ul>
|
||||
* <li>the closest matching configured {@code Serde<?>} bean if one exists</li>
|
||||
* <li>the Kafka Streams built-in serde if the target type is one of the built-in types exposed by Kafka Streams
|
||||
* (Integer, Long, Short, Double, Float, byte[], UUID and String)
|
||||
* <li>the fallback serde if specified and not one of the Kafka Streams exposed type serdes</li>
|
||||
* <li>the {@link JsonSerde} if the target type is not exactly {@code Object}</li>
|
||||
* <li>the fallback as the last resort</li>
|
||||
* </ul>
|
||||
* @param context the application context
|
||||
* @param targetType the target type to find the serde for
|
||||
* @param fallbackSerde the serde to use when no type can be inferred
|
||||
* @return serde to use for the target type or {@code fallbackSerde} as outlined in the method description
|
||||
*/
|
||||
static Serde<?> resolveForType(ConfigurableApplicationContext context, ResolvableType targetType, @Nullable Serde<?> fallbackSerde) {
|
||||
|
||||
Class<?> genericRawClazz = targetType.getRawClass();
|
||||
|
||||
// We don't attempt to find a matching Serde for type '?' - just return fallback
|
||||
if (genericRawClazz == null) {
|
||||
return fallbackSerde;
|
||||
}
|
||||
|
||||
List<String> matchingSerdes = beanNamesForMatchingSerdes(context, targetType);
|
||||
if (!matchingSerdes.isEmpty()) {
|
||||
return context.getBean(matchingSerdes.get(0), Serde.class);
|
||||
}
|
||||
|
||||
// Use standard serde for built-in types
|
||||
Serde<?> standardDefaultSerde = getStandardDefaultSerde(genericRawClazz);
|
||||
if (standardDefaultSerde != null) {
|
||||
return standardDefaultSerde;
|
||||
}
|
||||
|
||||
// Use fallback if specified and not from std defaults (we know from above that type is not std default
|
||||
// so using a fallback that is std default type would not work)
|
||||
if (fallbackSerde != null && !isSerdeFromStandardDefaults(fallbackSerde)) {
|
||||
return fallbackSerde;
|
||||
}
|
||||
|
||||
// Use JsonSerde if type is not exactly Object
|
||||
if (!genericRawClazz.isAssignableFrom((Object.class))) {
|
||||
return new JsonSerde<>(genericRawClazz);
|
||||
}
|
||||
|
||||
// Finally, just resort to using the fallback
|
||||
return fallbackSerde;
|
||||
}
|
||||
|
||||
private static Serde<?> getStandardDefaultSerde(Class<?> genericRawClazz) {
|
||||
try {
|
||||
return Serdes.serdeFrom(genericRawClazz);
|
||||
}
|
||||
catch (IllegalArgumentException ex) {
|
||||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace(ex);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static boolean isSerdeFromStandardDefaults(Serde<?> serde) {
|
||||
if (serde == null) {
|
||||
return false;
|
||||
}
|
||||
return STANDARD_SERDE_CLASSNAMES.contains(serde.getClass().getName());
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the names of all {@link Serde} beans that can be used for {@code targetType}.
|
||||
*
|
||||
* @param context the application context
|
||||
* @param targetType the target type the serdes are being matched for
|
||||
* @return list of bean names for matching serdes ordered by most specific match, or an empty list if no matches found
|
||||
*/
|
||||
static List<String> beanNamesForMatchingSerdes(ConfigurableApplicationContext context, ResolvableType targetType) {
|
||||
// We don't attempt to find a matching Serde for type '?'
|
||||
if (targetType.getRawClass() == null) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
List<SerdeWithSpecificityScore> matchingSerdes = new ArrayList<>();
|
||||
ResolvableType serdeType = ResolvableType.forClassWithGenerics(Serde.class, targetType);
|
||||
String[] serdeBeanNames = BeanFactoryUtils.beanNamesForTypeIncludingAncestors(context.getBeanFactory(),
|
||||
serdeType, false, false);
|
||||
Arrays.stream(serdeBeanNames).forEach((beanName) -> {
|
||||
try {
|
||||
BeanDefinition beanDefinition = context.getBeanFactory().getMergedBeanDefinition(beanName);
|
||||
ResolvableType serdeBeanGeneric = beanDefinition.getResolvableType().getGeneric(0);
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Found matching Serde<" + serdeBeanGeneric.getType() + "> under beanName=" + beanName);
|
||||
}
|
||||
matchingSerdes.add(new SerdeWithSpecificityScore(calculateScore(targetType, serdeBeanGeneric), beanName));
|
||||
}
|
||||
catch (Exception ex) {
|
||||
LOG.warn("Failed introspecting Serde bean '" + beanName + "'", ex);
|
||||
}
|
||||
});
|
||||
if (!matchingSerdes.isEmpty()) {
|
||||
return matchingSerdes.stream().sorted(Collections.reverseOrder())
|
||||
.map(SerdeWithSpecificityScore::getSerdeBeanName)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate a score to indicate how specific of a match one resolvable type is to another.
|
||||
* <p><br>Simple string comparison (the number of matching leading characters between two type strings) is used to
|
||||
* calculate the score. This approach avoids the recursive nature of the possible generic types, and leverages
|
||||
* the type strings returned from {@link ResolvableType#toString()} and {@link Type#getTypeName()} which already
|
||||
* include the properly handled generic types. The score is a composite of both of these properties because the
|
||||
* 'toString' value does not include bounds values and works as a tie-breaker to distinguish non-exact
|
||||
* matches that are closer in nature.
|
||||
* <p><br><b>Example:</b>
|
||||
* <pre>{@code
|
||||
* -------------------------------------------------------------------------------------------------------
|
||||
* targetType: Foo<Date> toString='Foo<Date>' typeName='Foo<java.util.Date>'
|
||||
* typeToCheck1: Foo<Date> toString='Foo<Date>' typeName='Foo<java.util.Date>'
|
||||
* typeToCheck2: Foo<? extends Date> toString='Foo<Date>' typeName='Foo<? extends java.util.Date>'
|
||||
* -------------------------------------------------------------------------------------------------------
|
||||
* }</pre>
|
||||
*
|
||||
* If using only the 'toString' value then both types would have the same score. However, including the 'typeName'
|
||||
* value in the score differentiates them - in this case it is clear that 'typeToCheck1' is a direct match.
|
||||
*
|
||||
* @param targetType the target type
|
||||
* @param typeToScore the type to calculate a score for
|
||||
* @return a score on how close of a match {@code typeToScore} is to {@code targetType} - the higher the score,
|
||||
* the closer the match
|
||||
*/
|
||||
private static int calculateScore(ResolvableType targetType, ResolvableType typeToScore) {
|
||||
int score = countLeadingMatchingChars(targetType.getType().getTypeName(), typeToScore.getType().getTypeName());
|
||||
score += countLeadingMatchingChars(targetType.toString(), typeToScore.toString());
|
||||
return score;
|
||||
}
|
||||
|
||||
private static int countLeadingMatchingChars(String s1, String s2) {
|
||||
if (s1 == null || s2 == null) {
|
||||
return 0;
|
||||
}
|
||||
int matchCount = 0;
|
||||
for (int i = 0; i < s1.length() && i < s2.length(); i++) {
|
||||
if (s1.charAt(i) != s2.charAt(i)) {
|
||||
break;
|
||||
}
|
||||
matchCount++;
|
||||
}
|
||||
return matchCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Private internal class used strictly to 'remember' a score for a serde and use it for sorting later.
|
||||
*/
|
||||
private static class SerdeWithSpecificityScore implements Comparable<SerdeWithSpecificityScore> {
|
||||
private final Integer score;
|
||||
private final String serdeBeanName;
|
||||
|
||||
SerdeWithSpecificityScore(Integer score, String serdeBeanName) {
|
||||
this.score = Objects.requireNonNull(score);
|
||||
this.serdeBeanName = Objects.requireNonNull(serdeBeanName);
|
||||
}
|
||||
|
||||
String getSerdeBeanName() {
|
||||
return serdeBeanName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(SerdeWithSpecificityScore other) {
|
||||
return this.score.compareTo(other.score);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021-2021 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.binder.kafka.streams;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
import org.apache.kafka.streams.errors.DeserializationExceptionHandler;
|
||||
import org.apache.kafka.streams.processor.ProcessorContext;
|
||||
|
||||
/**
|
||||
*
|
||||
* {@link DeserializationExceptionHandler} that allows to silently skip
|
||||
* deserialization exceptions and continue processing.
|
||||
*
|
||||
* @author Soby Chakco
|
||||
* @since 3.1.2
|
||||
*/
|
||||
public class SkipAndContinueExceptionHandler implements DeserializationExceptionHandler {
|
||||
|
||||
@Override
|
||||
public DeserializationExceptionHandler.DeserializationHandlerResponse handle(final ProcessorContext context,
|
||||
final ConsumerRecord<byte[], byte[]> record,
|
||||
final Exception exception) {
|
||||
return DeserializationExceptionHandler.DeserializationHandlerResponse.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configure(final Map<String, ?> configs) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user