SpringCloud/spring-cloud-stream
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-2311: Migrate AWS Kinesis binder to core

Browse Source 
Fixes https://github.com/spring-cloud/spring-cloud-stream/issues/2311

* Use Tescontainers for Localstack
* Fix tests according new `core` status quo
* Improve `KinesisStreamProvisioner` to wait for stream become active
before moving on to other logic

Disable Kinesis binder LocalstackContainerTest on mac
This commit is contained in:
Artem Bilan
2022-03-31 15:52:13 -04:00
committed by Soby Chacko
parent bf02424d27
commit d351afe27a
42 changed files with 5386 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
binders/kinesis-binder/.gitignore vendored Normal file
View File

@@ -0,0 +1,29 @@
apps/
/application.yml
/application.properties
asciidoctor.css
*~
.#*
*#
target/
build/
bin/
_site/
.classpath
.project
.settings
.springBeans
.DS_Store
*.sw*
*.iml
*.ipr
*.iws
.idea/
.factorypath
spring-xd-samples/*/xd
dump.rdb
coverage-error.log
.apt_generated
aws.credentials.properties
/spring-cloud-stream-binder-kinesis/nbproject/
nb-configuration.xml

1
binders/kinesis-binder/.mvn/maven.config Normal file
View File

@@ -0,0 +1 @@
-DaltSnapshotDeploymentRepository=repo.spring.io::default::https://repo.spring.io/libs-snapshot-local -P spring

117
binders/kinesis-binder/.mvn/wrapper/MavenWrapperDownloader.java vendored Normal file
View File

@@ -0,0 +1,117 @@
/*
* Copyright 2007-present the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.net.*;
import java.io.*;
import java.nio.channels.*;
import java.util.Properties;
public class MavenWrapperDownloader {
private static final String WRAPPER_VERSION = "0.5.6";
/**
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
*/
private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+ WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
/**
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
* use instead of the default one.
*/
private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
".mvn/wrapper/maven-wrapper.properties";
/**
* Path where the maven-wrapper.jar will be saved to.
*/
private static final String MAVEN_WRAPPER_JAR_PATH =
".mvn/wrapper/maven-wrapper.jar";
/**
* Name of the property which should be used to override the default download url for the wrapper.
*/
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
public static void main(String args[]) {
System.out.println("- Downloader started");
File baseDirectory = new File(args[0]);
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
// If the maven-wrapper.properties exists, read it and check if it contains a custom
// wrapperUrl parameter.
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
String url = DEFAULT_DOWNLOAD_URL;
if(mavenWrapperPropertyFile.exists()) {
FileInputStream mavenWrapperPropertyFileInputStream = null;
try {
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
Properties mavenWrapperProperties = new Properties();
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
} catch (IOException e) {
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
} finally {
try {
if(mavenWrapperPropertyFileInputStream != null) {
mavenWrapperPropertyFileInputStream.close();
}
} catch (IOException e) {
// Ignore ...
}
}
}
System.out.println("- Downloading from: " + url);
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
if(!outputFile.getParentFile().exists()) {
if(!outputFile.getParentFile().mkdirs()) {
System.out.println(
"- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
}
}
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
try {
downloadFileFromURL(url, outputFile);
System.out.println("Done");
System.exit(0);
} catch (Throwable e) {
System.out.println("- Error downloading");
e.printStackTrace();
System.exit(1);
}
}
private static void downloadFileFromURL(String urlString, File destination) throws Exception {
if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
String username = System.getenv("MVNW_USERNAME");
char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
Authenticator.setDefault(new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(username, password);
}
});
}
URL website = new URL(urlString);
ReadableByteChannel rbc;
rbc = Channels.newChannel(website.openStream());
FileOutputStream fos = new FileOutputStream(destination);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
rbc.close();
}
}

BIN
binders/kinesis-binder/.mvn/wrapper/maven-wrapper.jar vendored Normal file
View File

Binary file not shown.

2
binders/kinesis-binder/.mvn/wrapper/maven-wrapper.properties vendored Normal file
View File

@@ -0,0 +1,2 @@
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip
wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar

66
binders/kinesis-binder/.settings.xml Normal file
View File

@@ -0,0 +1,66 @@
<?xml version="1.0" encoding="UTF-8"?>
<settings>
<servers>
<server>
<id>repo.spring.io</id>
<username>${env.CI_DEPLOY_USERNAME}</username>
<password>${env.CI_DEPLOY_PASSWORD}</password>
</server>
</servers>
<profiles>
<profile>
<!--
N.B. this profile is only here to support users and IDEs that do not use Maven 3.3.
It isn't needed on the command line if you use the wrapper script (mvnw) or if you use
a native Maven with the right version. Eclipse users should points their Maven tooling to
this settings file, or copy the profile into their ~/.m2/settings.xml.
-->
<id>spring</id>
<activation><activeByDefault>true</activeByDefault></activation>
<repositories>
<repository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/libs-snapshot-local</url>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
<repository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/libs-milestone-local</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>spring-releases</id>
<name>Spring Releases</name>
<url>https://repo.spring.io/release</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/libs-snapshot-local</url>
<snapshots>
<enabled>true</enabled>
</snapshots>
</pluginRepository>
<pluginRepository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/libs-milestone-local</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</pluginRepository>
</pluginRepositories>
</profile>
</profiles>
</settings>

201
binders/kinesis-binder/LICENSE Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
https://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

245
binders/kinesis-binder/mvnw vendored Normal file
View File

@@ -0,0 +1,245 @@
#!/bin/sh
# ----------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Maven2 Start Up Batch script
#
# Required ENV vars:
# ------------------
# JAVA_HOME - location of a JDK home dir
#
# Optional ENV vars
# -----------------
# M2_HOME - location of maven2's installed home dir
# MAVEN_OPTS - parameters passed to the Java VM when running Maven
# e.g. to debug Maven itself, use
# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
# ----------------------------------------------------------------------------
if [ -z "$MAVEN_SKIP_RC" ] ; then
if [ -f /etc/mavenrc ] ; then
. /etc/mavenrc
fi
if [ -f "$HOME/.mavenrc" ] ; then
. "$HOME/.mavenrc"
fi
fi
# OS specific support. $var _must_ be set to either true or false.
cygwin=false;
darwin=false;
mingw=false
case "`uname`" in
CYGWIN*) cygwin=true ;;
MINGW*) mingw=true;;
Darwin*) darwin=true
#
# Look for the Apple JDKs first to preserve the existing behaviour, and then look
# for the new JDKs provided by Oracle.
#
if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then
#
# Apple JDKs
#
export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home
fi
if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then
#
# Apple JDKs
#
export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home
fi
if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then
#
# Oracle JDKs
#
export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home
fi
if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then
#
# Apple JDKs
#
export JAVA_HOME=`/usr/libexec/java_home`
fi
;;
esac
if [ -z "$JAVA_HOME" ] ; then
if [ -r /etc/gentoo-release ] ; then
JAVA_HOME=`java-config --jre-home`
fi
fi
if [ -z "$M2_HOME" ] ; then
## resolve links - $0 may be a link to maven's home
PRG="$0"
# need this for relative symlinks
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG="`dirname "$PRG"`/$link"
fi
done
saveddir=`pwd`
M2_HOME=`dirname "$PRG"`/..
# make it fully qualified
M2_HOME=`cd "$M2_HOME" && pwd`
cd "$saveddir"
# echo Using m2 at $M2_HOME
fi
# For Cygwin, ensure paths are in UNIX format before anything is touched
if $cygwin ; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --unix "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
fi
# For Migwn, ensure paths are in UNIX format before anything is touched
if $mingw ; then
[ -n "$M2_HOME" ] &&
M2_HOME="`(cd "$M2_HOME"; pwd)`"
[ -n "$JAVA_HOME" ] &&
JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
# TODO classpath?
fi
if [ -z "$JAVA_HOME" ]; then
javaExecutable="`which javac`"
if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
# readlink(1) is not available as standard on Solaris 10.
readLink=`which readlink`
if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
if $darwin ; then
javaHome="`dirname \"$javaExecutable\"`"
javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
else
javaExecutable="`readlink -f \"$javaExecutable\"`"
fi
javaHome="`dirname \"$javaExecutable\"`"
javaHome=`expr "$javaHome" : '\(.*\)/bin'`
JAVA_HOME="$javaHome"
export JAVA_HOME
fi
fi
fi
if [ -z "$JAVACMD" ] ; then
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
else
JAVACMD="`which java`"
fi
fi
if [ ! -x "$JAVACMD" ] ; then
echo "Error: JAVA_HOME is not defined correctly." >&2
echo " We cannot execute $JAVACMD" >&2
exit 1
fi
if [ -z "$JAVA_HOME" ] ; then
echo "Warning: JAVA_HOME environment variable is not set."
fi
CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
# For Cygwin, switch paths to Windows format before running java
if $cygwin; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --path --windows "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
fi
# traverses directory structure from process work directory to filesystem root
# first directory with .mvn subdirectory is considered project base directory
find_maven_basedir() {
local basedir=$(pwd)
local wdir=$(pwd)
while [ "$wdir" != '/' ] ; do
if [ -d "$wdir"/.mvn ] ; then
basedir=$wdir
break
fi
wdir=$(cd "$wdir/.."; pwd)
done
echo "${basedir}"
}
# concatenates all lines of a file
concat_lines() {
if [ -f "$1" ]; then
echo "$(tr -s '\n' ' ' < "$1")"
fi
}
export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)}
MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
# Provide a "standardized" way to retrieve the CLI args that will
# work with both Windows and non-Windows executions.
MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@"
export MAVEN_CMD_LINE_ARGS
WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
echo "Running version check"
VERSION=$( sed '\!<parent!,\!</parent!d' `dirname $0`/pom.xml | grep '<version' | head -1 | sed -e 's/.*<version>//' -e 's!</version>.*$!!' )
echo "The found version is [${VERSION}]"
if echo $VERSION | egrep -q 'M|RC'; then
echo Activating \"milestone\" profile for version=\"$VERSION\"
echo $MAVEN_ARGS | grep -q milestone || MAVEN_ARGS="$MAVEN_ARGS -Pmilestone"
else
echo Deactivating \"milestone\" profile for version=\"$VERSION\"
echo $MAVEN_ARGS | grep -q milestone && MAVEN_ARGS=$(echo $MAVEN_ARGS | sed -e 's/-Pmilestone//')
fi
exec "$JAVACMD" \
$MAVEN_OPTS \
-classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
"-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
${WRAPPER_LAUNCHER} ${MAVEN_ARGS} "$@"

143
binders/kinesis-binder/mvnw.cmd vendored Normal file
View File

@@ -0,0 +1,143 @@
@REM ----------------------------------------------------------------------------
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing,
@REM software distributed under the License is distributed on an
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@REM KIND, either express or implied. See the License for the
@REM specific language governing permissions and limitations
@REM under the License.
@REM ----------------------------------------------------------------------------
@REM ----------------------------------------------------------------------------
@REM Maven2 Start Up Batch script
@REM
@REM Required ENV vars:
@REM JAVA_HOME - location of a JDK home dir
@REM
@REM Optional ENV vars
@REM M2_HOME - location of maven2's installed home dir
@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending
@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
@REM e.g. to debug Maven itself, use
@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
@REM ----------------------------------------------------------------------------
@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
@echo off
@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on'
@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
@REM set %HOME% to equivalent of $HOME
if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
@REM Execute a user defined script before this one
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
@REM check for pre script, once with legacy .bat ending and once with .cmd ending
if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
:skipRcPre
@setlocal
set ERROR_CODE=0
@REM To isolate internal variables from possible post scripts, we use another setlocal
@setlocal
@REM ==== START VALIDATION ====
if not "%JAVA_HOME%" == "" goto OkJHome
echo.
echo Error: JAVA_HOME not found in your environment. >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
:OkJHome
if exist "%JAVA_HOME%\bin\java.exe" goto init
echo.
echo Error: JAVA_HOME is set to an invalid directory. >&2
echo JAVA_HOME = "%JAVA_HOME%" >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
@REM ==== END VALIDATION ====
:init
@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
@REM Fallback to current working directory if not found.
set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
set EXEC_DIR=%CD%
set WDIR=%EXEC_DIR%
:findBaseDir
IF EXIST "%WDIR%"\.mvn goto baseDirFound
cd ..
IF "%WDIR%"=="%CD%" goto baseDirNotFound
set WDIR=%CD%
goto findBaseDir
:baseDirFound
set MAVEN_PROJECTBASEDIR=%WDIR%
cd "%EXEC_DIR%"
goto endDetectBaseDir
:baseDirNotFound
set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
cd "%EXEC_DIR%"
:endDetectBaseDir
IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
@setlocal EnableExtensions EnableDelayedExpansion
for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
:endReadAdditionalConfig
SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
if ERRORLEVEL 1 goto error
goto end
:error
set ERROR_CODE=1
:end
@endlocal & set ERROR_CODE=%ERROR_CODE%
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
@REM check for post script, once with legacy .bat ending and once with .cmd ending
if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
:skipRcPost
@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
if "%MAVEN_BATCH_PAUSE%" == "on" pause
if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
exit /B %ERROR_CODE%

244
binders/kinesis-binder/pom.xml Normal file
View File

@@ -0,0 +1,244 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-build</artifactId>
<version>4.0.0-SNAPSHOT</version>
<relativePath/>
</parent>
<artifactId>spring-cloud-stream-binder-kinesis-parent</artifactId>
<version>4.0.0-SNAPSHOT</version>
<packaging>pom</packaging>
<properties>
<spring-cloud-stream.version>4.0.0-SNAPSHOT</spring-cloud-stream.version>
<java.version>17</java.version>
<spring-cloud-aws.version>2.4.0</spring-cloud-aws.version>
<spring-integration-aws.version>3.0.0-SNAPSHOT</spring-integration-aws.version>
<dynamodb-lock-client.version>1.1.0</dynamodb-lock-client.version>
<amazon-kinesis-client.version>1.14.8</amazon-kinesis-client.version>
<amazon-kinesis-producer.version>0.14.12</amazon-kinesis-producer.version>
<dynamodb-stream.version>1.5.3</dynamodb-stream.version>
<testcontainers.version>1.16.3</testcontainers.version>
</properties>
<modules>
<module>spring-cloud-stream-binder-kinesis</module>
<module>spring-cloud-stream-binder-kinesis-docs</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-binder-kinesis</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream</artifactId>
<version>${spring-cloud-stream.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.integration</groupId>
<artifactId>spring-integration-aws</artifactId>
<version>${spring-integration-aws.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>dynamodb-lock-client</artifactId>
<version>${dynamodb-lock-client.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>dynamodb-streams-kinesis-adapter</artifactId>
<version>${dynamodb-stream.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>amazon-kinesis-client</artifactId>
<version>${amazon-kinesis-client.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>amazon-kinesis-producer</artifactId>
<version>${amazon-kinesis-producer.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-binder-test</artifactId>
<version>${spring-cloud-stream.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-test-support-internal</artifactId>
<version>${spring-cloud-stream.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.awspring.cloud</groupId>
<artifactId>spring-cloud-aws-dependencies</artifactId>
<version>${spring-cloud-aws.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers-bom</artifactId>
<version>${testcontainers.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
<quiet>true</quiet>
</configuration>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin.version}</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
<compilerArgument>-parameters</compilerArgument>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
</plugin>
</plugins>
</build>
<reporting>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
</plugin>
</plugins>
</reporting>
<profiles>
<profile>
<id>spring</id>
<repositories>
<repository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/snapshot</url>
<snapshots>
<enabled>true</enabled>
</snapshots>
<releases>
<enabled>false</enabled>
</releases>
</repository>
<repository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/milestone</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>spring-releases</id>
<name>Spring Releases</name>
<url>https://repo.spring.io/release</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/snapshot</url>
<snapshots>
<enabled>true</enabled>
</snapshots>
<releases>
<enabled>false</enabled>
</releases>
</pluginRepository>
<pluginRepository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/milestone</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</pluginRepository>
<pluginRepository>
<id>spring-releases</id>
<name>Spring Releases</name>
<url>https://repo.spring.io/release</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</pluginRepository>
</pluginRepositories>
</profile>
</profiles>
<repositories>
<repository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/snapshot</url>
</repository>
<repository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/milestone</url>
</repository>
<repository>
<id>spring-releases</id>
<name>Spring Releases</name>
<url>https://repo.spring.io/release</url>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/snapshot</url>
</pluginRepository>
<pluginRepository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/milestone</url>
</pluginRepository>
<pluginRepository>
<id>spring-releases</id>
<name>Spring Releases</name>
<url>https://repo.spring.io/release</url>
</pluginRepository>
</pluginRepositories>
</project>

65
binders/kinesis-binder/spring-cloud-stream-binder-kinesis-docs/pom.xml Normal file
View File

@@ -0,0 +1,65 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>spring-cloud-stream-binder-kinesis-docs</artifactId>
<parent>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-binder-kinesis-parent</artifactId>
<version>4.0.0-SNAPSHOT</version>
</parent>
<packaging>jar</packaging>
<name>spring-cloud-stream-binder-kinesis-docs</name>
<description>Spring Cloud Stream AWS Kinesis Binder Docs</description>
<properties>
<docs.main>spring-cloud-stream-binder-kinesis</docs.main>
<main.basedir>${basedir}/..</main.basedir>
<maven.plugin.plugin.version>3.4</maven.plugin.plugin.version>
<configprops.inclusionPattern>.*stream.*</configprops.inclusionPattern>
<upload-docs-zip.phase>deploy</upload-docs-zip.phase>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>spring-cloud-stream-binder-kinesis</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/asciidoc</sourceDirectory>
</build>
<profiles>
<profile>
<id>docs</id>
<build>
<plugins>
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.asciidoctor</groupId>
<artifactId>asciidoctor-maven-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

20
binders/kinesis-binder/spring-cloud-stream-binder-kinesis-docs/src/main/asciidoc/Guardfile Normal file
View File

@@ -0,0 +1,20 @@
require 'asciidoctor'
require 'erb'
guard 'shell' do
watch(/.*\.adoc$/) {|m|
Asciidoctor.render_file('index.adoc', \
:in_place => true, \
:safe => Asciidoctor::SafeMode::UNSAFE, \
:attributes=> { \
'source-highlighter' => 'prettify', \
'icons' => 'font', \
'linkcss'=> 'true', \
'copycss' => 'true', \
'doctype' => 'book'})
}
end
guard 'livereload' do
watch(%r{^.+\.(css|js|html)$})
end

59
binders/kinesis-binder/spring-cloud-stream-binder-kinesis-docs/src/main/asciidoc/building.adoc Normal file
View File

@@ -0,0 +1,59 @@
[[building]]
== Building
:jdkversion: 1.7
=== Basic Compile and Test
To build the source you will need to install JDK {jdkversion}.
The build uses the Maven wrapper so you don't have to install a specific version of Maven.
To enable the tests, you should have https://github.com/mhart/kinesalite[Kinesalite] server running before building.
The main build command is
----
$ ./mvnw clean install
----
You can also add `-DskipTests` if you like, to avoid running the tests.
NOTE: You can also install Maven (>=3.3.3) yourself and run the `mvn` command in place of `./mvnw` in the examples below.
If you do that you also might need to add `-P spring` if your local Maven settings do not contain repository declarations for spring pre-release artifacts.
NOTE: Be aware that you might need to increase the amount of memory available to Maven by setting a `MAVEN_OPTS` environment variable with a value like `-Xmx512m -XX:MaxPermSize=128m`.
We try to cover this in the `.mvn` configuration, so if you find you have to do it to make a build succeed, please raise a ticket to get the settings added to source control.
The projects that require middleware generally include a `docker-compose.yml`, so consider using https://docs.docker.com/compose[Docker Compose] to run the middleware servers in Docker containers.
=== Documentation
There is a "full" profile that will generate documentation.
=== Working with the code
If you don't have an IDE preference we would recommend that you use https://www.springsource.com/developer/sts[Spring Tools Suite] or https://eclipse.org[Eclipse] when working with the code.
We use the https://eclipse.org/m2e/[m2eclipe] eclipse plugin for maven support.
Other IDEs and tools should also work without issue.
==== Importing into eclipse with m2eclipse
We recommend the https://eclipse.org/m2e/[m2eclipe] eclipse plugin when working with eclipse.
If you don't already have m2eclipse installed it is available from the "eclipse marketplace".
Unfortunately m2e does not yet support Maven 3.3, so once the projects are imported into Eclipse you will also need to tell m2eclipse to use the `.settings.xml` file for the projects.
If you do not do this you may see many different errors related to the POMs in the projects.
Open your Eclipse preferences, expand the Maven preferences, and select User Settings.
In the User Settings field click Browse and navigate to the Spring Cloud project you imported selecting the `.settings.xml` file in that project.
Click Apply and then OK to save the preference changes.
NOTE: Alternatively you can copy the repository settings from https://github.com/spring-cloud/spring-cloud-build/blob/main/.settings.xml[`.settings.xml`] into your own `~/.m2/settings.xml`.
==== Importing into eclipse without m2eclipse
If you prefer not to use m2eclipse you can generate eclipse project metadata using the following command:
[indent=0]
----
$ ./mvnw eclipse:eclipse
----
The generated eclipse projects can be imported by selecting `import existing projects` from the `file` menu.

26
binders/kinesis-binder/spring-cloud-stream-binder-kinesis-docs/src/main/asciidoc/contributing.adoc Normal file
View File

@@ -0,0 +1,26 @@
[[contributing]]
== Contributing
Spring Cloud is released under the non-restrictive Apache 2.0 license, and follows a very standard Github development process, using Github tracker for issues and merging pull requests into main.
If you want to contribute even something trivial please do not hesitate, but follow the guidelines below.
=== Sign the Contributor License Agreement
Before we accept a non-trivial patch or pull request we will need you to sign the https://support.springsource.com/spring_committer_signup[contributor's agreement].
Signing the contributor's agreement does not grant anyone commit rights to the main repository, but it does mean that we can accept your contributions, and you will get an author credit if we do.
Active contributors might be asked to join the core team, and given the ability to merge pull requests.
=== Code Conventions and Housekeeping
None of these is essential for a pull request, but they will all help.
They can also be added after the original pull request but before a merge.
* Use the Spring Framework code format conventions. If you use Eclipse you can import formatter settings using the `eclipse-code-formatter.xml` file from the https://github.com/spring-cloud/build/tree/main/eclipse-coding-conventions.xml[Spring Cloud Build] project.
If using IntelliJ, you can use the https://plugins.jetbrains.com/plugin/6546[Eclipse Code Formatter Plugin] to import the same file.
* Make sure all new `.java` files to have a simple Javadoc class comment with at least an `@author` tag identifying you, and preferably at least a paragraph on what the class is for.
* Add the ASF license header comment to all new `.java` files (copy from existing files in the project)
* Add yourself as an `@author` to the .java files that you modify substantially (more than cosmetic changes).
* Add some Javadocs and, if you change the namespace, some XSD doc elements.
* A few unit tests would help a lot as well -- someone has to do it.
* If no-one else is using your branch, please rebase it against the current main (or other target branch in the main project).
* When writing a commit message please follow https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html[these conventions], if you are fixing an existing issue please add `Fixes gh-XXXX` at the end of the commit message (where XXXX is the issue number).

BIN
binders/kinesis-binder/spring-cloud-stream-binder-kinesis-docs/src/main/asciidoc/images/kinesis-binder.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

14
binders/kinesis-binder/spring-cloud-stream-binder-kinesis-docs/src/main/asciidoc/index-docinfo.xml Normal file
View File

@@ -0,0 +1,14 @@
<productname>Spring Cloud Stream Kinesis Binder</productname>
<releaseinfo>{spring-cloud-stream-binder-kinesis-version}</releaseinfo>
<copyright>
<year>2017-2021</year>
<holder>VMWare, Inc.</holder>
</copyright>
<legalnotice>
<para>
Copies of this document may be made for your own use and for distribution to
others, provided that you do not charge any fee for such copies and further
provided that each copy contains this Copyright Notice, whether distributed in
print or electronically.
</para>
</legalnotice>

34
binders/kinesis-binder/spring-cloud-stream-binder-kinesis-docs/src/main/asciidoc/index.adoc Normal file
View File

@@ -0,0 +1,34 @@
[[spring-cloud-stream-binder-kinesis-reference]]
= Spring Cloud Stream AWS Kinesis Binder Reference Guide
Artem Bilan
:doctype: book
:toc:
:toclevels: 4
:source-highlighter: prettify
:numbered:
:icons: font
:hide-uri-scheme:
:spring-cloud-stream-binder-kinesis-repo: snapshot
:github-tag: main
:spring-cloud-stream-binder-kinesis-docs-version: current
:spring-cloud-stream-binder-kinesis-docs: https://docs.spring.io/spring-cloud-stream-binder-kinesis/docs/{spring-cloud-stream-binder-kinesis-docs-version}/reference
:spring-cloud-stream-binder-kinesis-docs-current: https://docs.spring.io/spring-cloud-stream-binder-kinesis/docs/current-SNAPSHOT/reference/html/
:github-repo: spring-cloud/spring-cloud-stream-binder-aws-kinesis
:github-raw: https://raw.github.com/{github-repo}/{github-tag}
:github-code: https://github.com/{github-repo}/tree/{github-tag}
:github-wiki: https://github.com/{github-repo}/wiki
:github-main-code: https://github.com/{github-repo}/tree/main
:sc-ext: java
// ======================================================================================
= Reference Guide
include::overview.adoc[]
= Appendices
[appendix]
include::building.adoc[]
[appendix]
include::contributing.adoc[]
// ======================================================================================

544
binders/kinesis-binder/spring-cloud-stream-binder-kinesis-docs/src/main/asciidoc/overview.adoc Normal file
View File

@@ -0,0 +1,544 @@
[partintro]
--
This guide describes the https://aws.amazon.com/kinesis/[AWS Kinesis] implementation of the Spring Cloud Stream Binder.
It contains information about its design, usage and configuration options, as well as information on how the Stream Cloud Stream concepts map into AWS Kinesis specific constructs.
--
== Usage
For using the AWS Kinesis Binder, you just need to add it to your Spring Cloud Stream application, using the following Maven coordinates:
[source,xml]
----
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-binder-kinesis</artifactId>
</dependency>
----
== Kinesis Binder Overview
The Spring Cloud Stream Binder for AWS Kinesis provides the binding implementation for the Spring Cloud Stream.
This implementation uses Spring Integration AWS Kinesis Channel Adapters at its foundation.
The following captures how the Kinesis Binder implementation maps each of the configured destination to a AWS Kinesis Streams:
.Kinesis Binder
image::images/kinesis-binder.png[width=300,scaledwidth="50%"]
Unlike https://kafka.apache.org/[Apache Kafka] the AWS Kinesis doesn't provide out-of-the-box support for consumer groups.
The support of this feature is implemented as a part of `MetadataStore` key for shard checkpoints in the `KinesisMessageDrivenChannelAdapter` - `[CONSUMER_GROUP]:[STREAM]:[SHARD_ID]`.
In addition the `LockRegistry` is used to ensure exclusive access to each shard.
This way only one channel adapter in the same consumer group will consumer messages from a single shard in the stream it is configured for.
The partitioning logic in AWS Kinesis is similar to the Apache Kafka support, but with slightly different logic.
The `partitionKey` on the producer side determines which shard in the stream the data record is assigned to.
Partition keys are Unicode strings with a maximum length limit of 256 characters for each key.
AWS Kinesis uses the partition key as input to a hash function that maps the partition key and associated data to a specific shard.
Specifically, an MD5 hash function is used to map partition keys to 128-bit integer values and to map associated data records to shards.
As a result of this hashing mechanism, all data records with the same partition key map to the same shard within the stream.
But at the same time we can't select target shard to send explicitly.
Although calculating the hash manually (and use `explicitHashKeyExpression` for producer, respectively), we may track the target shard by inclusion into its `HashKeyRange`.
By default partition key is a result of the `Object.hash()` from the message `payload`.
The Spring Cloud Stream partition handling logic is excluded in case of AWS Kinesis Binder since it is out of use and the provided `producer.partitionKeyExpression` is propagated to the `KinesisMessageHandler` directly.
On the consumer side the `instanceCount` and `instanceIndex` are used to distribute shards between consumers in group evenly.
This has an effect only for regular `KinesisMessageDrivenChannelAdapter` which can assign specific shards for the target Kinesis consumer.
With Kinesis Client Library we can only subscriber to the provided stream and shards distribution is done by that client.
See more information in the Kinesis Client Library https://docs.aws.amazon.com/streams/latest/dev/developing-consumers-with-kcl.html[documentation].
== Consumer Groups
Consumer groups are implemented with focus on High availability, Message ordering and guaranteed Message delivery in Spring cloud stream.
A `single consumer` for the message is ensured by https://docs.spring.io/spring-cloud-stream/docs/Elmhurst.RELEASE/reference/htmlsingle/#consumer-groups[consumer group abstraction].
To have a highly available consumer group for your kinesis stream:
- Ensure all instances of your consumer applications use a shared `DynamoDbMetadataStore` and `DynamoDbLockRegistry` (See below for configuration options).
- Use same group name for the channel in all application instances by using property `spring.cloud.stream.bindings.<bindingTarget>.group`.
These configurations alone guarantee HA, message ordering and guaranteed message delivery.
However, even distribution across instances is not guaranteed as of now.
There is a very high chance that a single instance in a consumer group will pick up all the shards for consuming.
But, when that instance goes down (couldn't send heartbeat for any reason), other instance in the consumer group will start processing from the last checkpoint of the previous consumer (for shardIterator type TRIM_HORIZON).
So, configuring consumer concurrency is important to achieve throughput.
It can be configured using `spring.cloud.stream.bindings.<bindingTarget>.consumer.concurrency`.
=== Static shard distribution within a single consumer group
It is possible to evenly distribute shard across all instances within a single consumer group.
This done by configuring:
- `spring.cloud.stream.instanceCount=` to number of instances
- `spring.cloud.stream.instanceIndex=` current instance's index
The only way to achieve HA in this case is that, when an instance processing a particular shard goes down, another instance must have `spring.cloud.stream.instanceIndex=` to be the same as the failed instance's index to start processing from those shards.
== Configuration Options
This section contains settings specific to the Kinesis Binder and bound channels.
For general binding configuration options and properties, please refer to the https://docs.spring.io/spring-cloud-stream/docs/current/reference/htmlsingle/#_configuration_options[Spring Cloud Stream core documentation].
[[kinesis-binder-properties]]
=== Kinesis Binder Properties
The following properties are available for Kinesis Binder configuration, which start with the `spring.cloud.stream.kinesis.binder.` prefix
headers::
The set of custom headers to transfer over AWS Kinesis
+
Default: "correlationId", "sequenceSize", "sequenceNumber", "contentType", "originalContentType".
describeStreamBackoff::
The amount of time in milliseconds in between retries for the `DescribeStream` operation
+
Default: `1000`.
describeStreamRetries::
The amount of times the consumer will retry a `DescribeStream` operation waiting for the stream to be in `ACTIVE` state
+
Default: `50`.
autoCreateStream::
If set to `true`, the binder will create the stream automatically.
If set to `false`, the binder will rely on the stream being already created.
+
Default: `true`
autoAddShards::
If set to `true`, the binder will create new shards automatically.
If set to `false`, the binder will rely on the shard size of the stream being already configured.
If the shard count of the target stream is smaller than the expected value, the binder will ignore that value
+
Default: `false`
minShardCount::
Effective only if `autoAddShards` is set to `true`.
The minimum number of shards that the binder will configure on the stream from which it produces/consumes data.
It can be superseded by the `partitionCount` setting of the producer or by the value of `instanceCount * concurrency` settings of the producer (if either is larger)
+
Default: `1`
kplKclEnabled::
Enable the usage of https://docs.aws.amazon.com/streams/latest/dev/developing-consumers-with-kcl.html[Kinesis Client Library] / https://docs.aws.amazon.com/streams/latest/dev/developing-producers-with-kpl.html[Kinesis Producer Library] for all message consumption and production
+
Default: `false`
=== MetadataStore
Support for consumer groups is implemented using https://github.com/spring-projects/spring-integration-aws#metadata-store-for-amazon-dynamodb[DynamoDbMetadataStore].
The `partitionKey` name used in the table is `KEY`.
This is not configurable.
DynamoDB Checkpoint properties are prefixed with `spring.cloud.stream.kinesis.binder.checkpoint.`
table::
The name to give the DynamoDb table
+
Default: `SpringIntegrationMetadataStore`
createDelay::
The amount of time in seconds between each polling attempt while waiting for the checkpoint DynamoDB table to be created
+
Default: `1`
createRetries::
The amount of times the consumer will poll DynamoDB while waiting for the checkpoint table to be created
+
Default: `25`
billingMode::
The Billing Mode of the DynamoDB table. See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ReadWriteCapacityMode.html#HowItWorks.OnDemand[DynamoDB On-Demand Mode]. Possible values are `provisioned` and `payPerRequest`. If left empty or set to `payPerRequest` both `readCapacity` and `writeCapacity` are ignored
+
Default: `payPerRequest`
readCapacity::
The Read capacity of the DynamoDb table.
See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ReadWriteCapacityMode.html#HowItWorks.ProvisionedThroughput.Manual[DynamoDB Provisioned Throughput]. This property is used only when `billingMode` is set to `provisioned`
+
Default: `1`
writeCapacity::
The write capacity of the DynamoDb table.
See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ReadWriteCapacityMode.html#HowItWorks.ProvisionedThroughput.Manual[DynamoDB Provisioned Throughput]. This property is used only when `billingMode` is set to `provisioned`
+
Default: `1`
timeToLive::
A period in seconds for items expiration.
See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/TTL.html[DynamoDB TTL]
+
No default - means no records expiration.
=== LockRegistry
LockRegistry is used to ensure exclusive access to each shard so that, only one channel adapter in the same consumer group will consumer messages from a single shard in the stream.
This is implemented using https://github.com/spring-projects/spring-integration-aws#lock-registry-for-amazon-dynamodb[DynamoDbLockRegistry]
DynamoDB `LockRegistry` properties are prefixed with `spring.cloud.stream.kinesis.binder.locks.`
table::
The name to give the DynamoDB table
+
Default: `SpringIntegrationLockRegistry`
billingMode::
The Billing Mode of the DynamoDB table. See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ReadWriteCapacityMode.html#HowItWorks.OnDemand[DynamoDB On-Demand Mode]. Possible values are `provisioned` and `payPerRequest`. If left empty or set to `payPerRequest` both `readCapacity` and `writeCapacity` are ignored
+
Default: `payPerRequest`
readCapacity::
The Read capacity of the DynamoDB table.
See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ReadWriteCapacityMode.html#HowItWorks.ProvisionedThroughput.Manual[DynamoDB Provisioned Throughput]. This property is used only when `billingMode` is set to `provisioned`
+
Default: `1`
writeCapacity::
The write capacity of the DynamoDb table.
See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ReadWriteCapacityMode.html#HowItWorks.ProvisionedThroughput.Manual[DynamoDB Provisioned Throughput]. This property is used only when `billingMode` is set to `provisioned`
+
Default: `1`
leaseDuration::
The length of time that the lease for the lock will be granted for.
If this is set to, for example, 30 seconds, then the lock will expire if the heartbeat is not sent for at least 30 seconds (which would happen if the box or the heartbeat thread dies, for example.)
+
Default: `20`
heartbeatPeriod::
How often to update DynamoDB to note that the instance is still running (recommendation is to make this at least 3 times smaller than the `leaseDuration` - for example `heartBeatPeriod=1` second, `leaseDuration=10` seconds could be a reasonable configuration, make sure to include a buffer for network latency.)
+
Default: `5`
refreshPeriod::
How long to wait before trying to get the lock again (if set to 10 seconds, for example, it would attempt to do so every 10 seconds)
+
Default: `1000`
partitionKey::
The partition key name of the table.
+
Default: `lockKey`
sortKeyName::
The sort key name for DynamoDB table partitioning.
+
Default: `sortKey`
sortKey::
The sort key to try and acquire the lock on (specify if and only if the table has sort keys)
+
Default: `SpringIntegrationLocks`
=== Kinesis Consumer Properties
The following properties are available for Kinesis consumers only and must be prefixed with `spring.cloud.stream.kinesis.bindings.<channel-name>.consumer`
startTimeout::
The amount of time to wait for the consumer to start, in milliseconds.
+
Default: `60000`.
listenerMode::
The mode in which records are processed.
If `record`, each `Message` will contain `byte[]` from a single `Record.data`.
If `batch`, each `Message` will contain a `List<byte[]>` extracted from the consumed records.
When `useNativeDecoding = true` is used on the consumer together with the `listenerMode = batch`, there is no any out-of-the-box conversion happened and a result message contains a payload like `List<com.amazonaws.services.kinesis.model.Record>`.
It's up to target application to convert those records manually.
+
Default: `record`
checkpointMode::
The mode in which checkpoints are updated.
If `record`, checkpoints occur after each record is processed (but this option is only effective if `listenerMode` is set to `record`). If `batch`, checkpoints occur after each batch of records is processed.
If `manual`, checkpoints occur on demand via the `Checkpointer` callback.
If `periodic`, checkpoints occurs at specified time interval (from `interval` property in checkpoint configuration)
+
Default: `batch`
checkpointInterval::
The interval, in milliseconds, between two checkpoints when checkpoint mode is `periodic`.
+
Default - `5000`
workerId::
The worker identifier used to distinguish different workers/processes (only used when Kinesis Client Library is enabled).
+
No default - if not set, default value inside spring-integration-aws will be used (random UUID).
recordsLimit::
The maximum number of records to poll per `GetRecords` request.
Must not be greater than `10000`.
+
Default: `10000`
idleBetweenPolls::
The sleep interval used in the main loop between shards polling cycles, in milliseconds. Must not be less than `250`.
+
Default: `1000`
consumerBackoff::
The amount of time the consumer will wait to attempt another `GetRecords` operation after a read with no results, in milliseconds.
+
Default: `1000`
shardIteratorType::
The `com.amazonaws.services.kinesis.model.ShardIteratorType` name with an optional `sequenceNumber` for the `AT_SEQUENCE_NUMBER/AFTER_SEQUENCE_NUMBER` or milliseconds for the `AT_TIMESTAMP` after `:`.
For example: `AT_TIMESTAMP:1515090166767`.
+
Default: `LATEST` for anonymous groups and `TRIM_HORIZON` otherwise.
NOTE: When `TRIM_HORIZON` shard iterator type is used, we need to take into account the time lag which happens during pointing the `ShardIterator` to the last untrimmed record in the shard in the system (the oldest data record in the shard).
So the `getRecords()` will move from that point to the last point, which takes time.
It is by default 1 day and it can be extended to 7 days.
This happens only for new consumer groups.
Any subsequent starts of the consumer in the same group are adjusted according the stored checkpoint via `AFTER_SEQUENCE_NUMBER` iterator type.
dynamoDbStreams::
The `boolean` flag indicating that Kinesis consumer channel adapter should adapt DynamoDB Streams functionality instead of regular Kinesis streams.
The `spring.cloud.stream.bindings.<bindingTarget>.destination` value must be a DynamoDB table name.
Default: `false`.
Starting with version 2.0.1, beans of `KinesisClientLibConfiguration` type can be provided in the application context to have a full control over Kinesis Client Library configuration options.
The stream and consumer group (plus workerId) must be provided in the respective `KinesisClientLibConfiguration` bean.
When `KclMessageDrivenChannelAdapter` endpoint is configured in the binder, it selects an appropriate `KinesisClientLibConfiguration` from the application context according a destination (stream) for binding.
If there is no `KinesisClientLibConfiguration` bean for its stream, the `KclMessageDrivenChannelAdapter` falls back to original configuration with defaults options for its internal `KinesisClientLibConfiguration`.
shardId::
An explicit shard id to consume from.
NOTE: Kinesis Client Library does not support a configuration for a specific shard.
When `shardId` property is used, it is ignored for Kinesis Client Library and standard stream consumer distribution is applied.
Also, in case of an `instanceCount > 1`, application will throw validation exception.
The `instanceCount` and `shardId` are considered as mutually exclusive.
=== Kinesis Producer Properties
The following properties are available for Kinesis producers only and must be prefixed with `spring.cloud.stream.kinesis.bindings.<bindingTarget>.producer.`.
sync::
Whether the producer should act in a synchronous manner with respect to writing records into a stream.
If true, the producer will wait for a response from Kinesis after a `PutRecord` operation.
+
Default: `false`
sendTimeout::
Effective only if `sync` is set to `true`. The amount of time to wait for a response from Kinesis after a `PutRecord` operation, in milliseconds.
+
Default: `10000`
Also, if you'd like to produce a batch of records into Kinesis stream, the message payload must be as a `PutRecordsRequest` instance and general Spring Cloud Stream producer property `useNativeEncoding` must be set to `true`, so Spring Cloud Stream won't try to convert a `PutRecordsRequest` into a `byte[]`. The content of the `PutRecordsRequest` is now end-user responsibility.
[[kinesis-error-channels]]
== Error Channels
The binder can be configured to send producer exceptions to an error channel.
See https://docs.spring.io/spring-cloud-stream/docs/current/reference/htmlsingle/#_spring_integration_error_channel_support[the section on Spring Cloud error channel support] for more information.
The payload of the `ErrorMessage` for a send failure is an `AwsRequestFailureException` with properties:
* `failedMessage` - the spring-messaging `Message<?>` that failed to be sent.
* `request` - the raw `AmazonWebServiceRequest` (either `PutRecordRequest` or `PutRecordsRequest`) that was created from the `failedMessage`.
There is no automatic handling of these exceptions (such as sending to a dead letter queue), but you can consume these exceptions with your own Spring Integration flow.
[[dynamodb-streams]]
== DynamoDB Streams
Starting with version 1.2, the `KinesisMessageChannelBinder` supports DynamoDB Streams for Kinesis consumer channel adapter.
The functionality is based on the https://github.com/awslabs/dynamodb-streams-kinesis-adapter[DynamoDB Streams Kinesis Adapter] and every Kinesis consumer endpoint selects an appropriate Kinesis Client according the `spring.cloud.stream.kinesis.bindings.<bindingTarget>.consumer.dynamoDbStreams` boolean flag.
In this case a value for `spring.cloud.stream.bindings.<bindingTarget>.destination` must be a DynamoDB table name.
[[optional-resources]]
== Optional Resources
Starting with version 1.2, if your Spring Cloud Stream application delivered only in the `source` role, the extra beans, required for `sink` (or Kinesis consumers), are not going to be registered in the application context and, therefore, no need to worry about their resources on AWS.
The story is about DynamoDB and Cloud Watch.
[[aws-roles-and-policies]]
== AWS Roles and Policies
In order to be able to run properly on AWS, the role that will be used by the application needs to have a set of policies configured.
Here are the policies statements that your application role need:
[source,json]
----
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"kinesis:SubscribeToShard",
"kinesis:DescribeStreamSummary",
"kinesis:DescribeStreamConsumer",
"kinesis:GetShardIterator",
"kinesis:GetRecords",
"kinesis:PutRecords",
"kinesis:DescribeStream"
],
"Resource": [
"arn:aws:kinesis:<region>:<account_number>:*/*/consumer/*:*",
"arn:aws:kinesis:<region>:<account_number>:stream/<stream_name>"
]
},
{
"Effect": "Allow",
"Action": "kinesis:DescribeLimits",
"Resource": "*"
},
{
"Sid": "DynamoDB",
"Effect": "Allow",
"Action": [
"dynamodb:BatchGetItem",
"dynamodb:BatchWriteItem",
"dynamodb:PutItem",
"dynamodb:GetItem",
"dynamodb:Scan",
"dynamodb:Query",
"dynamodb:UpdateItem"
],
"Resource": [
"arn:aws:dynamodb:<region>:<account>:table/<name-of-metadata-table>",
"arn:aws:dynamodb:<region>:<account>:table/<name-of-lock-table>"
]
}
]
}
----
Keep in mind that these are only the policies to allow the application to consume/produce records from/to Kinesis.
If you're going to allow spring-cloud-stream-binder-kinesis to create the resources for you, you'll need an extra set of policies.
[source,json]
----
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"dynamodb:CreateTable",
"kinesis:CreateStream",
"kinesis:UpdateShardCount",
"kinesis:EnableEnhancedMonitoring",
"kinesis:DisableEnhancedMonitoring",
"dynamodb:DeleteTable",
"dynamodb:UpdateTable"
],
"Resource": [
"arn:aws:dynamodb:<region>:<account>:table/<table_name>",
"arn:aws:kinesis:<region>:<account>:stream/<stream_name>"
]
}
]
}
----
[[running-locally-with-localstack]]
== Running locally with localstack
Sometimes we don't have the necessary permissions to connect to the real Kinesis and DynamoDB from our developer's machine.
In moments like this, it's pretty useful to setup Localstack in your project, so you can run everything locally, without
having to worry about permissions and enterprise restrictions.
Create a *docker-compose.yaml* file, in the root of your project, to quickly start localstack
[source, yaml]
----
version: '3.5'
services:
localstack:
image: localstack/localstack:0.12.10
environment:
- AWS_DEFAULT_REGION=sa-east-1
- EDGE_PORT=4566
- SERVICES=kinesis, dynamodb
ports:
- '4566:4566'
volumes:
- localstack:/tmp/localstack
- './setup-localstack.sh:/docker-entrypoint-initaws.d/setup-localstack.sh'
volumes:
localstack:
----
After that, create a script called *setup-localstack.sh*, in the root directory, that will contain the script to create the
Kinesis Stream, and the 2 DynamoDB Tables
[source, shell script]
----
awslocal kinesis create-stream --stream-name my-test-stream --shard-count 1
awslocal dynamodb create-table \
--table-name spring-stream-lock-registry \
--attribute-definitions AttributeName=lockKey,AttributeType=S AttributeName=sortKey,AttributeType=S \
--key-schema AttributeName=lockKey,KeyType=HASH AttributeName=sortKey,KeyType=RANGE \
--provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5 \
--tags Key=Owner,Value=localstack
awslocal dynamodb create-table \
--table-name spring-stream-metadata \
--attribute-definitions AttributeName=KEY,AttributeType=S \
--key-schema AttributeName=KEY,KeyType=HASH \
--provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5 \
--tags Key=Owner,Value=localstack
awslocal dynamodb list-tables
awslocal kinesis list-streams
----
Since this file is being mapped to the localstack image, the container will automatically run this script the first
time you run the container.
To run localstack, just execute
[source, shell script]
----
docker-compose up -d
----
Your local AWS Endpoint is now available at http://localhost:4566
To put records into your test stream, just run
[source, shell script]
----
aws --endpoint-url=http://localhost:4566 kinesis put-record --stream-name my-test-stream --partition-key 1 --data <base64-encoded-data>
----
[[telling-the-binder-to-use-your-local-endpoint]]
=== Telling the binder to use your local endpoint
By default, the Kinesis and DynamoDB Client will try to hit the real AWS Endpoint. To change this behavior,
you have to declare a new @Bean, and override the endpoint.
For example:
[source, kotlin]
----
@Configuration
@Profile("local")
class DynamoDBConfigLocal {
@Value("\${cloud.aws.region.static}")
val region: String = ""
private val endpointUrl: String = "http://localhost:4566"
@Bean
@Primary
fun amazonDynamoDBAsync(): AmazonDynamoDBAsync {
return AmazonDynamoDBAsyncClientBuilder.standard()
.withEndpointConfiguration(AwsClientBuilder.EndpointConfiguration(this.endpointUrl, region))
.build()
}
}
----
[source, kotlin]
----
@Configuration
@Profile("local")
class KinesisConfigLocal {
@Value("\${cloud.aws.region.static}")
val region: String = ""
private val endpointUrl: String = "http://localhost:4566"
@Bean
fun amazonKinesis(awsCredentialsProvider: AWSCredentialsProvider): AmazonKinesisAsync {
return AmazonKinesisAsyncClientBuilder
.standard()
.withCredentials(awsCredentialsProvider)
.withEndpointConfiguration(AwsClientBuilder.EndpointConfiguration(endpointUrl, region))
.build()
}
}
----
Now, remember to pass the following environment variables when running locally:
[source, shell script]
----
SPRING_PROFILES_ACTIVE=local AWS_CBOR_DISABLE=true gradle bootRun
----
This will make sure that these beans are only instantiated when running locally, and will also disable CBOR, which is not
supported for the localstack's kinesis stream.
[[health-indicator]]
== Kinesis Binder Health Indicator
Version 2.0 has introduced a `KinesisBinderHealthIndicator` implementation which is a part of `BindersHealthContributor` composition under the `binders` path.
An out-of-the-box implementation iterates over Kinesis streams involved in the binder configuration calling a `describeStream` command against them.
If any of streams doesn't exist the health is treated as `DOWN`.
If `LimitExceededException` is thrown according `describeStream` limitations, the `KinesisBinderHealthIndicator` tries over again after one second interval.
Ony when all the configured stream are described properly the `UP` health is returned.
You can override out-of-the-box implementation provided your own bean with the `kinesisBinderHealthIndicator` name.

37
binders/kinesis-binder/spring-cloud-stream-binder-kinesis-docs/src/main/ruby/generate_readme.sh Normal file
View File

@@ -0,0 +1,37 @@
#!/usr/bin/env ruby
base_dir = File.join(File.dirname(__FILE__),'../../..')
src_dir = File.join(base_dir, "/src/main/asciidoc")
require 'asciidoctor'
require 'optparse'
options = {}
file = "#{src_dir}/README.adoc"
OptionParser.new do |o|
o.on('-o OUTPUT_FILE', 'Output file (default is stdout)') { |file| options[:to_file] = file unless file=='-' }
o.on('-h', '--help') { puts o; exit }
o.parse!
end
file = ARGV[0] if ARGV.length>0
# Copied from https://github.com/asciidoctor/asciidoctor-extensions-lab/blob/master/scripts/asciidoc-coalescer.rb
doc = Asciidoctor.load_file file, safe: :unsafe, header_only: true, attributes: options[:attributes]
header_attr_names = (doc.instance_variable_get :@attributes_modified).to_a
header_attr_names.each {|k| doc.attributes[%(#{k}!)] = '' unless doc.attr? k }
attrs = doc.attributes
attrs['allow-uri-read'] = true
puts attrs
out = "// Do not edit this file (e.g. go instead to src/main/asciidoc)\n\n"
doc = Asciidoctor.load_file file, safe: :unsafe, parse: false, attributes: attrs
out << doc.reader.read
unless options[:to_file]
puts out
else
File.open(options[:to_file],'w+') do |file|
file.write(out)
end
end

92
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/pom.xml Normal file
View File

@@ -0,0 +1,92 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-binder-kinesis-parent</artifactId>
<version>4.0.0-SNAPSHOT</version>
</parent>
<artifactId>spring-cloud-stream-binder-kinesis</artifactId>
<packaging>jar</packaging>
<name>spring-cloud-stream-binder-kinesis</name>
<description>AWS Kinesis Binder Implementation</description>
<dependencies>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>amazon-kinesis-client</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>amazon-kinesis-producer</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.integration</groupId>
<artifactId>spring-integration-aws</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream</artifactId>
</dependency>
<dependency>
<groupId>io.awspring.cloud</groupId>
<artifactId>spring-cloud-starter-aws</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-dynamodb</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>dynamodb-lock-client</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-kinesis</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>dynamodb-streams-kinesis-adapter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-binder-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-test-support-internal</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>localstack</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

70
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/KinesisBinderHealthIndicator.java Normal file
View File

@@ -0,0 +1,70 @@
/*
* Copyright 2017-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import com.amazonaws.services.kinesis.AmazonKinesisAsync;
import com.amazonaws.services.kinesis.model.LimitExceededException;
import com.amazonaws.services.kinesis.model.ListShardsRequest;
import org.springframework.boot.actuate.health.Health;
import org.springframework.boot.actuate.health.HealthIndicator;
/**
* @author Artem Bilan
*
* @since 2.0
*/
public class KinesisBinderHealthIndicator implements HealthIndicator {
private final KinesisMessageChannelBinder kinesisMessageChannelBinder;
public KinesisBinderHealthIndicator(KinesisMessageChannelBinder kinesisMessageChannelBinder) {
this.kinesisMessageChannelBinder = kinesisMessageChannelBinder;
}
@Override
public Health health() {
AmazonKinesisAsync amazonKinesis = this.kinesisMessageChannelBinder.getAmazonKinesis();
List<String> streamsInUse = new ArrayList<>(this.kinesisMessageChannelBinder.getStreamsInUse());
for (String stream : streamsInUse) {
while (true) {
try {
amazonKinesis.listShards(new ListShardsRequest().withStreamName(stream).withMaxResults(1));
break;
}
catch (LimitExceededException ex) {
try {
TimeUnit.SECONDS.sleep(1);
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
return Health.down(ex).build();
}
}
catch (Exception ex) {
return Health.down(ex).build();
}
}
}
return Health.up().build();
}
}

548
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/KinesisMessageChannelBinder.java Normal file
View File

@@ -0,0 +1,548 @@
/*
* Copyright 2017-2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.cloudwatch.AmazonCloudWatch;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBStreams;
import com.amazonaws.services.dynamodbv2.model.DescribeTableResult;
import com.amazonaws.services.dynamodbv2.streamsadapter.AmazonDynamoDBStreamsAdapterClient;
import com.amazonaws.services.kinesis.AmazonKinesis;
import com.amazonaws.services.kinesis.AmazonKinesisAsync;
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream;
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration;
import com.amazonaws.services.kinesis.model.InvalidArgumentException;
import com.amazonaws.services.kinesis.model.Shard;
import com.amazonaws.services.kinesis.model.ShardIteratorType;
import com.amazonaws.services.kinesis.producer.KinesisProducer;
import com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration;
import org.springframework.cloud.stream.binder.AbstractMessageChannelBinder;
import org.springframework.cloud.stream.binder.BinderHeaders;
import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider;
import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder;
import org.springframework.cloud.stream.binder.PartitionKeyExtractorStrategy;
import org.springframework.cloud.stream.binder.kinesis.adapter.SpringDynamoDBAdapterClient;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisBinderConfigurationProperties;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisConsumerProperties;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisExtendedBindingProperties;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisProducerProperties;
import org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisConsumerDestination;
import org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner;
import org.springframework.cloud.stream.provisioning.ConsumerDestination;
import org.springframework.cloud.stream.provisioning.ProducerDestination;
import org.springframework.cloud.stream.provisioning.ProvisioningException;
import org.springframework.expression.EvaluationContext;
import org.springframework.integration.aws.inbound.kinesis.KclMessageDrivenChannelAdapter;
import org.springframework.integration.aws.inbound.kinesis.KinesisMessageDrivenChannelAdapter;
import org.springframework.integration.aws.inbound.kinesis.KinesisMessageHeaderErrorMessageStrategy;
import org.springframework.integration.aws.inbound.kinesis.KinesisShardOffset;
import org.springframework.integration.aws.outbound.AbstractAwsMessageHandler;
import org.springframework.integration.aws.outbound.KinesisMessageHandler;
import org.springframework.integration.aws.outbound.KplMessageHandler;
import org.springframework.integration.core.MessageProducer;
import org.springframework.integration.expression.ExpressionUtils;
import org.springframework.integration.expression.FunctionExpression;
import org.springframework.integration.metadata.ConcurrentMetadataStore;
import org.springframework.integration.support.ErrorMessageStrategy;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.integration.support.locks.LockRegistry;
import org.springframework.lang.Nullable;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.MessageHandler;
import org.springframework.messaging.support.ChannelInterceptor;
import org.springframework.messaging.support.InterceptableChannel;
import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils;
import org.springframework.util.ObjectUtils;
import org.springframework.util.StringUtils;
/**
*
* The Spring Cloud Stream Binder implementation for AWS Kinesis.
*
* @author Peter Oates
* @author Artem Bilan
* @author Arnaud Lecollaire
* @author Dirk Bonhomme
* @author Asiel Caballero
* @author Dmytro Danilenkov
*
*/
public class KinesisMessageChannelBinder extends
AbstractMessageChannelBinder<ExtendedConsumerProperties<KinesisConsumerProperties>,
ExtendedProducerProperties<KinesisProducerProperties>, KinesisStreamProvisioner>
implements
ExtendedPropertiesBinder<MessageChannel, KinesisConsumerProperties, KinesisProducerProperties> {
private static final ErrorMessageStrategy ERROR_MESSAGE_STRATEGY = new KinesisMessageHeaderErrorMessageStrategy();
private final List<String> streamsInUse = new ArrayList<>();
private final KinesisBinderConfigurationProperties configurationProperties;
private final AmazonDynamoDBStreamsAdapterClient dynamoDBStreamsAdapter;
private final AmazonKinesisAsync amazonKinesis;
private final AWSCredentialsProvider awsCredentialsProvider;
private final AmazonCloudWatch cloudWatchClient;
private final AmazonDynamoDB dynamoDBClient;
private KinesisExtendedBindingProperties extendedBindingProperties = new KinesisExtendedBindingProperties();
@Nullable
private ConcurrentMetadataStore checkpointStore;
@Nullable
private LockRegistry lockRegistry;
@Nullable
private KinesisProducerConfiguration kinesisProducerConfiguration;
private EvaluationContext evaluationContext;
private List<KinesisClientLibConfiguration> kinesisClientLibConfigurations;
public KinesisMessageChannelBinder(KinesisBinderConfigurationProperties configurationProperties,
KinesisStreamProvisioner provisioningProvider, AmazonKinesisAsync amazonKinesis,
AWSCredentialsProvider awsCredentialsProvider,
@Nullable AmazonDynamoDB dynamoDBClient,
@Nullable AmazonDynamoDBStreams dynamoDBStreams,
@Nullable AmazonCloudWatch cloudWatchClient) {
super(headersToMap(configurationProperties), provisioningProvider);
Assert.notNull(amazonKinesis, "'amazonKinesis' must not be null");
Assert.notNull(awsCredentialsProvider, "'awsCredentialsProvider' must not be null");
this.configurationProperties = configurationProperties;
this.amazonKinesis = amazonKinesis;
this.cloudWatchClient = cloudWatchClient;
this.dynamoDBClient = dynamoDBClient;
this.awsCredentialsProvider = awsCredentialsProvider;
if (dynamoDBStreams != null) {
this.dynamoDBStreamsAdapter = new SpringDynamoDBAdapterClient(dynamoDBStreams);
}
else {
this.dynamoDBStreamsAdapter = null;
}
}
public void setExtendedBindingProperties(KinesisExtendedBindingProperties extendedBindingProperties) {
this.extendedBindingProperties = extendedBindingProperties;
}
public void setCheckpointStore(ConcurrentMetadataStore checkpointStore) {
this.checkpointStore = checkpointStore;
}
public void setLockRegistry(LockRegistry lockRegistry) {
this.lockRegistry = lockRegistry;
}
public void setKinesisProducerConfiguration(KinesisProducerConfiguration kinesisProducerConfiguration) {
this.kinesisProducerConfiguration = kinesisProducerConfiguration;
}
public void setKinesisClientLibConfigurations(List<KinesisClientLibConfiguration> kinesisClientLibConfigurations) {
this.kinesisClientLibConfigurations = kinesisClientLibConfigurations;
}
@Override
public KinesisConsumerProperties getExtendedConsumerProperties(String channelName) {
return this.extendedBindingProperties.getExtendedConsumerProperties(channelName);
}
@Override
public KinesisProducerProperties getExtendedProducerProperties(String channelName) {
return this.extendedBindingProperties.getExtendedProducerProperties(channelName);
}
@Override
public String getDefaultsPrefix() {
return this.extendedBindingProperties.getDefaultsPrefix();
}
@Override
public Class<? extends BinderSpecificPropertiesProvider> getExtendedPropertiesEntryClass() {
return this.extendedBindingProperties.getExtendedPropertiesEntryClass();
}
public AmazonKinesisAsync getAmazonKinesis() {
return this.amazonKinesis;
}
public List<String> getStreamsInUse() {
return this.streamsInUse;
}
@Override
protected void onInit() throws Exception {
super.onInit();
this.evaluationContext = ExpressionUtils.createStandardEvaluationContext(getBeanFactory());
}
@Override
protected MessageHandler createProducerMessageHandler(ProducerDestination destination,
ExtendedProducerProperties<KinesisProducerProperties> producerProperties,
MessageChannel errorChannel) {
FunctionExpression<Message<?>> partitionKeyExpression =
new FunctionExpression<>((m) ->
m.getHeaders().containsKey(BinderHeaders.PARTITION_HEADER)
? m.getHeaders().get(BinderHeaders.PARTITION_HEADER)
: m.getPayload().hashCode());
final AbstractAwsMessageHandler<?> messageHandler;
if (this.configurationProperties.isKplKclEnabled()) {
messageHandler = createKplMessageHandler(destination, partitionKeyExpression);
}
else {
messageHandler = createKinesisMessageHandler(destination, partitionKeyExpression);
}
messageHandler.setSync(producerProperties.getExtension().isSync());
messageHandler.setSendTimeout(producerProperties.getExtension().getSendTimeout());
messageHandler.setFailureChannel(errorChannel);
messageHandler.setBeanFactory(getBeanFactory());
this.streamsInUse.add(destination.getName());
return messageHandler;
}
private AbstractAwsMessageHandler<?> createKinesisMessageHandler(ProducerDestination destination,
FunctionExpression<Message<?>> partitionKeyExpression) {
final KinesisMessageHandler messageHandler;
messageHandler = new KinesisMessageHandler(this.amazonKinesis);
messageHandler.setStream(destination.getName());
messageHandler.setPartitionKeyExpression(partitionKeyExpression);
return messageHandler;
}
private AbstractAwsMessageHandler<?> createKplMessageHandler(ProducerDestination destination,
FunctionExpression<Message<?>> partitionKeyExpression) {
final KplMessageHandler messageHandler;
messageHandler = new KplMessageHandler(new KinesisProducer(this.kinesisProducerConfiguration));
messageHandler.setStream(destination.getName());
messageHandler.setPartitionKeyExpression(partitionKeyExpression);
return messageHandler;
}
@Override
protected void postProcessOutputChannel(MessageChannel outputChannel,
ExtendedProducerProperties<KinesisProducerProperties> producerProperties) {
if (outputChannel instanceof InterceptableChannel && producerProperties.isPartitioned()) {
((InterceptableChannel) outputChannel)
.addInterceptor(0,
new ChannelInterceptor() {
private final PartitionKeyExtractorStrategy partitionKeyExtractorStrategy;
{
if (StringUtils.hasText(producerProperties.getPartitionKeyExtractorName())) {
this.partitionKeyExtractorStrategy =
getBeanFactory()
.getBean(producerProperties.getPartitionKeyExtractorName(),
PartitionKeyExtractorStrategy.class);
}
else {
this.partitionKeyExtractorStrategy =
(message) ->
producerProperties.getPartitionKeyExpression()
.getValue(KinesisMessageChannelBinder.this.evaluationContext,
message);
}
}
@Override
public Message<?> preSend(Message<?> message, MessageChannel channel) {
Object partitionKey = this.partitionKeyExtractorStrategy.extractKey(message);
return MessageBuilder.fromMessage(message)
.setHeader(BinderHeaders.PARTITION_OVERRIDE, partitionKey)
.build();
}
});
}
}
@Override
protected MessageProducer createConsumerEndpoint(ConsumerDestination destination,
String group,
ExtendedConsumerProperties<KinesisConsumerProperties> properties) {
ConsumerDestination destinationToUse = destination;
if (properties.getExtension().isDynamoDbStreams()) {
DescribeTableResult describeTableResult = this.dynamoDBClient.describeTable(destinationToUse.getName());
String latestStreamArn = describeTableResult.getTable().getLatestStreamArn();
if (StringUtils.hasText(latestStreamArn)) {
destinationToUse = new KinesisConsumerDestination(latestStreamArn, Collections.emptyList());
}
else {
throw new ProvisioningException("The DynamoDB table ["
+ destinationToUse.getName()
+ "] doesn't have Streams enabled.");
}
}
else {
this.streamsInUse.add(destinationToUse.getName());
}
MessageProducer adapter;
if (this.configurationProperties.isKplKclEnabled()) {
adapter = createKclConsumerEndpoint(destinationToUse, group, properties);
}
else {
adapter = createKinesisConsumerEndpoint(destinationToUse, group, properties);
}
return adapter;
}
private MessageProducer createKclConsumerEndpoint(ConsumerDestination destination, String group,
ExtendedConsumerProperties<KinesisConsumerProperties> properties) {
KinesisConsumerProperties kinesisConsumerProperties = properties.getExtension();
if (kinesisConsumerProperties.getShardId() != null) {
logger.warn("Kinesis Client Library doesn't does not support explicit shard configuration. " +
"Ignoring 'shardId' property");
}
String shardIteratorType = kinesisConsumerProperties.getShardIteratorType();
AmazonKinesis amazonKinesisClient =
kinesisConsumerProperties.isDynamoDbStreams()
? this.dynamoDBStreamsAdapter
: this.amazonKinesis;
String stream = destination.getName();
KinesisClientLibConfiguration kinesisClientLibConfiguration = obtainKinesisClientLibConfiguration(stream, group);
KclMessageDrivenChannelAdapter adapter;
String consumerGroup;
if (kinesisClientLibConfiguration == null) {
adapter = new KclMessageDrivenChannelAdapter(stream, amazonKinesisClient, this.cloudWatchClient,
this.dynamoDBClient, this.awsCredentialsProvider);
boolean anonymous = !StringUtils.hasText(group);
consumerGroup = anonymous ? "anonymous." + UUID.randomUUID() : group;
adapter.setConsumerGroup(consumerGroup);
if (StringUtils.hasText(shardIteratorType)) {
adapter.setStreamInitialSequence(InitialPositionInStream.valueOf(shardIteratorType));
}
adapter.setIdleBetweenPolls(kinesisConsumerProperties.getIdleBetweenPolls());
adapter.setConsumerBackoff(kinesisConsumerProperties.getConsumerBackoff());
if (kinesisConsumerProperties.getWorkerId() != null) {
adapter.setWorkerId(kinesisConsumerProperties.getWorkerId());
}
}
else {
adapter = new KclMessageDrivenChannelAdapter(kinesisClientLibConfiguration, amazonKinesisClient,
this.cloudWatchClient, this.dynamoDBClient);
consumerGroup = kinesisClientLibConfiguration.getApplicationName();
}
adapter.setCheckpointMode(kinesisConsumerProperties.getCheckpointMode());
adapter.setCheckpointsInterval(kinesisConsumerProperties.getCheckpointInterval());
adapter.setListenerMode(kinesisConsumerProperties.getListenerMode());
if (properties.isUseNativeDecoding()) {
adapter.setConverter(null);
}
else {
// Defer byte[] conversion to the InboundContentTypeConvertingInterceptor
adapter.setConverter((bytes) -> bytes);
}
ErrorInfrastructure errorInfrastructure = registerErrorInfrastructure(destination, consumerGroup, properties);
adapter.setErrorMessageStrategy(ERROR_MESSAGE_STRATEGY);
adapter.setErrorChannel(errorInfrastructure.getErrorChannel());
adapter.setBindSourceRecord(true);
return adapter;
}
private KinesisClientLibConfiguration obtainKinesisClientLibConfiguration(String stream, String group) {
KinesisClientLibConfiguration candidate = null;
for (KinesisClientLibConfiguration conf : this.kinesisClientLibConfigurations) {
if (stream.equals(conf.getStreamName())) {
candidate = conf;
if (Objects.equals(group, conf.getApplicationName())) {
break;
}
}
}
return candidate;
}
private MessageProducer createKinesisConsumerEndpoint(ConsumerDestination destination, String group,
ExtendedConsumerProperties<KinesisConsumerProperties> properties) {
KinesisConsumerProperties kinesisConsumerProperties = properties.getExtension();
if (properties.getInstanceCount() > 1 && properties.getExtension().getShardId() != null) {
throw new InvalidArgumentException("'instanceCount' more than 1 and 'shardId' cannot be provided together.");
}
Set<KinesisShardOffset> shardOffsets = null;
String shardIteratorType = kinesisConsumerProperties.getShardIteratorType();
KinesisShardOffset kinesisShardOffset = KinesisShardOffset.latest();
if (StringUtils.hasText(shardIteratorType)) {
String[] typeValue = shardIteratorType.split(":", 2);
ShardIteratorType iteratorType = ShardIteratorType.valueOf(typeValue[0]);
kinesisShardOffset = new KinesisShardOffset(iteratorType);
if (typeValue.length > 1) {
if (ShardIteratorType.AT_TIMESTAMP.equals(iteratorType)) {
kinesisShardOffset
.setTimestamp(new Date(Long.parseLong(typeValue[1])));
}
else {
kinesisShardOffset.setSequenceNumber(typeValue[1]);
}
}
}
if (properties.getInstanceCount() > 1) {
shardOffsets = new HashSet<>();
KinesisConsumerDestination kinesisConsumerDestination = (KinesisConsumerDestination) destination;
List<Shard> shards = kinesisConsumerDestination.getShards();
for (int i = 0; i < shards.size(); i++) {
// divide shards across instances
if ((i % properties.getInstanceCount()) == properties.getInstanceIndex()) {
KinesisShardOffset shardOffset = new KinesisShardOffset(kinesisShardOffset);
shardOffset.setStream(destination.getName());
shardOffset.setShard(shards.get(i).getShardId());
shardOffsets.add(shardOffset);
}
}
}
KinesisMessageDrivenChannelAdapter adapter;
AmazonKinesis amazonKinesisClient =
kinesisConsumerProperties.isDynamoDbStreams()
? this.dynamoDBStreamsAdapter
: this.amazonKinesis;
String shardId = kinesisConsumerProperties.getShardId();
if (CollectionUtils.isEmpty(shardOffsets) && shardId == null) {
adapter = new KinesisMessageDrivenChannelAdapter(amazonKinesisClient, destination.getName());
}
else if (shardId != null) {
KinesisShardOffset shardOffset = new KinesisShardOffset(kinesisShardOffset);
shardOffset.setStream(destination.getName());
shardOffset.setShard(shardId);
adapter = new KinesisMessageDrivenChannelAdapter(amazonKinesisClient, shardOffset);
}
else {
adapter = new KinesisMessageDrivenChannelAdapter(amazonKinesisClient,
shardOffsets.toArray(new KinesisShardOffset[0]));
}
boolean anonymous = !StringUtils.hasText(group);
String consumerGroup = anonymous ? "anonymous." + UUID.randomUUID() : group;
adapter.setConsumerGroup(consumerGroup);
adapter.setStreamInitialSequence(
anonymous || StringUtils.hasText(shardIteratorType) ? kinesisShardOffset
: KinesisShardOffset.trimHorizon());
adapter.setListenerMode(kinesisConsumerProperties.getListenerMode());
if (properties.isUseNativeDecoding()) {
adapter.setConverter(null);
}
else {
// Defer byte[] conversion to the InboundContentTypeConvertingInterceptor
adapter.setConverter((bytes) -> bytes);
}
adapter.setCheckpointMode(kinesisConsumerProperties.getCheckpointMode());
adapter.setRecordsLimit(kinesisConsumerProperties.getRecordsLimit());
adapter.setIdleBetweenPolls(kinesisConsumerProperties.getIdleBetweenPolls());
adapter.setConsumerBackoff(kinesisConsumerProperties.getConsumerBackoff());
adapter.setCheckpointsInterval(kinesisConsumerProperties.getCheckpointInterval());
if (this.checkpointStore != null) {
adapter.setCheckpointStore(this.checkpointStore);
}
adapter.setLockRegistry(this.lockRegistry);
adapter.setConcurrency(properties.getConcurrency());
adapter.setStartTimeout(kinesisConsumerProperties.getStartTimeout());
adapter.setDescribeStreamBackoff(
this.configurationProperties.getDescribeStreamBackoff());
adapter.setDescribeStreamRetries(
this.configurationProperties.getDescribeStreamRetries());
ErrorInfrastructure errorInfrastructure = registerErrorInfrastructure(destination,
consumerGroup, properties);
adapter.setErrorMessageStrategy(ERROR_MESSAGE_STRATEGY);
adapter.setErrorChannel(errorInfrastructure.getErrorChannel());
adapter.setBindSourceRecord(true);
return adapter;
}
@Override
protected ErrorMessageStrategy getErrorMessageStrategy() {
return ERROR_MESSAGE_STRATEGY;
}
private static String[] headersToMap(KinesisBinderConfigurationProperties configurationProperties) {
Assert.notNull(configurationProperties,
"'configurationProperties' must not be null");
if (ObjectUtils.isEmpty(configurationProperties.getHeaders())) {
return BinderHeaders.STANDARD_HEADERS;
}
else {
String[] combinedHeadersToMap = Arrays.copyOfRange(
BinderHeaders.STANDARD_HEADERS, 0,
BinderHeaders.STANDARD_HEADERS.length
+ configurationProperties.getHeaders().length);
System.arraycopy(configurationProperties.getHeaders(), 0,
combinedHeadersToMap, BinderHeaders.STANDARD_HEADERS.length,
configurationProperties.getHeaders().length);
return combinedHeadersToMap;
}
}
}

100
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/adapter/SpringDynamoDBAdapterClient.java Normal file
View File

@@ -0,0 +1,100 @@
/*
* Copyright 2020-2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis.adapter;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBStreams;
import com.amazonaws.services.dynamodbv2.model.AmazonDynamoDBException;
import com.amazonaws.services.dynamodbv2.streamsadapter.AmazonDynamoDBStreamsAdapterClient;
import com.amazonaws.services.kinesis.model.DescribeStreamRequest;
import com.amazonaws.services.kinesis.model.InvalidArgumentException;
import com.amazonaws.services.kinesis.model.ListShardsRequest;
import com.amazonaws.services.kinesis.model.ListShardsResult;
import com.amazonaws.services.kinesis.model.ResourceNotFoundException;
import com.amazonaws.services.kinesis.model.StreamDescription;
/**
* This is Spring Cloud DynamoDB Adapter to be able to support {@code ListShards} operations.
*
* @author Asiel Caballero
*
* @since 2.0.3
*
* @see <a href="https://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListShards.html">ListShards</a>
*/
public class SpringDynamoDBAdapterClient extends AmazonDynamoDBStreamsAdapterClient {
private static final String SEPARATOR = "!!##%%";
public SpringDynamoDBAdapterClient(AmazonDynamoDBStreams amazonDynamoDBStreams) {
super(amazonDynamoDBStreams);
}
/**
* List shards for a DynamoDB Stream using its {@code DescribeStream} API, as they don't support
* {@code ListShards} operations. Returns the result adapted to use the AmazonKinesis model.
* @param request Container for the necessary parameters to execute the ListShards service method
* @return The response from the DescribeStream service method, adapted for use with the AmazonKinesis model
*/
@Override
public ListShardsResult listShards(ListShardsRequest request) {
try {
if (request.getNextToken() != null && request.getStreamName() != null) {
throw new InvalidArgumentException("NextToken and StreamName cannot be provided together.");
}
String streamName = request.getStreamName();
String exclusiveStartShardId = request.getExclusiveStartShardId();
if (request.getNextToken() != null) {
String[] split = request.getNextToken().split(SEPARATOR);
if (split.length != 2) {
throw new InvalidArgumentException("Invalid ShardIterator");
}
streamName = split[0];
exclusiveStartShardId = split[1];
}
DescribeStreamRequest dsr = new DescribeStreamRequest()
.withStreamName(streamName)
.withExclusiveStartShardId(exclusiveStartShardId)
.withLimit(request.getMaxResults());
StreamDescription streamDescription = describeStream(dsr).getStreamDescription();
ListShardsResult result = new ListShardsResult()
.withShards(streamDescription.getShards());
if (streamDescription.getHasMoreShards()) {
result.withNextToken(buildFakeNextToken(streamName,
streamDescription.getShards().get(streamDescription.getShards().size() - 1).getShardId()));
}
return result;
}
catch (AmazonDynamoDBException ex) {
ResourceNotFoundException resourceEx = new ResourceNotFoundException(ex.getMessage());
resourceEx.setStackTrace(ex.getStackTrace());
throw resourceEx;
}
}
private String buildFakeNextToken(String streamName, String lastShard) {
return lastShard != null ? streamName + SEPARATOR + lastShard : null;
}
}

263
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/config/KinesisBinderConfiguration.java Normal file
View File

@@ -0,0 +1,263 @@
/*
* Copyright 2017-2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis.config;
import java.util.List;
import java.util.Set;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.cloudwatch.AmazonCloudWatch;
import com.amazonaws.services.cloudwatch.AmazonCloudWatchAsync;
import com.amazonaws.services.cloudwatch.AmazonCloudWatchAsyncClientBuilder;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsyncClientBuilder;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBStreams;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBStreamsClientBuilder;
import com.amazonaws.services.kinesis.AmazonKinesisAsync;
import com.amazonaws.services.kinesis.AmazonKinesisAsyncClientBuilder;
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration;
import com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration;
import io.awspring.cloud.autoconfigure.context.ContextCredentialsAutoConfiguration;
import io.awspring.cloud.autoconfigure.context.ContextRegionProviderAutoConfiguration;
import io.awspring.cloud.core.region.RegionProvider;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.actuate.autoconfigure.health.ConditionalOnEnabledHealthIndicator;
import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.stream.binder.Binder;
import org.springframework.cloud.stream.binder.kinesis.KinesisBinderHealthIndicator;
import org.springframework.cloud.stream.binder.kinesis.KinesisMessageChannelBinder;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisBinderConfigurationProperties;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisExtendedBindingProperties;
import org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner;
import org.springframework.cloud.stream.binding.Bindable;
import org.springframework.cloud.stream.config.ConsumerEndpointCustomizer;
import org.springframework.cloud.stream.config.ProducerMessageHandlerCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.integration.aws.lock.DynamoDbLockRegistry;
import org.springframework.integration.aws.metadata.DynamoDbMetadataStore;
import org.springframework.integration.aws.outbound.AbstractAwsMessageHandler;
import org.springframework.integration.endpoint.MessageProducerSupport;
import org.springframework.integration.metadata.ConcurrentMetadataStore;
import org.springframework.integration.support.locks.LockRegistry;
/**
* The auto-configuration for AWS components and Spring Cloud Stream Kinesis Binder.
*
* @author Peter Oates
* @author Artem Bilan
* @author Arnaud Lecollaire
* @author Asiel Caballero
*/
@Configuration(proxyBeanMethods = false)
@ConditionalOnMissingBean(Binder.class)
@EnableConfigurationProperties({ KinesisBinderConfigurationProperties.class, KinesisExtendedBindingProperties.class })
@Import({ ContextCredentialsAutoConfiguration.class, ContextRegionProviderAutoConfiguration.class })
public class KinesisBinderConfiguration {
private final KinesisBinderConfigurationProperties configurationProperties;
private final AWSCredentialsProvider awsCredentialsProvider;
private final String region;
private final boolean hasInputs;
public KinesisBinderConfiguration(KinesisBinderConfigurationProperties configurationProperties,
AWSCredentialsProvider awsCredentialsProvider,
RegionProvider regionProvider,
List<Bindable> bindables) {
this.configurationProperties = configurationProperties;
this.awsCredentialsProvider = awsCredentialsProvider;
this.region = regionProvider.getRegion().getName();
this.hasInputs =
bindables.stream()
.map(Bindable::getInputs)
.flatMap(Set::stream)
.findFirst()
.isPresent();
}
@Bean
@ConditionalOnMissingBean
public AmazonKinesisAsync amazonKinesis() {
return AmazonKinesisAsyncClientBuilder.standard()
.withCredentials(this.awsCredentialsProvider)
.withRegion(this.region)
.build();
}
@Bean
public KinesisStreamProvisioner provisioningProvider(AmazonKinesisAsync amazonKinesis) {
return new KinesisStreamProvisioner(amazonKinesis, this.configurationProperties);
}
@Bean
@ConditionalOnMissingBean
public AmazonDynamoDBAsync dynamoDB() {
if (this.hasInputs) {
return AmazonDynamoDBAsyncClientBuilder.standard()
.withCredentials(this.awsCredentialsProvider)
.withRegion(this.region)
.build();
}
else {
return null;
}
}
@Bean
@ConditionalOnMissingBean
@ConditionalOnBean(AmazonDynamoDBAsync.class)
@ConditionalOnProperty(name = "spring.cloud.stream.kinesis.binder.kpl-kcl-enabled", havingValue = "false",
matchIfMissing = true)
public LockRegistry dynamoDBLockRegistry(@Autowired(required = false) AmazonDynamoDBAsync dynamoDB) {
if (dynamoDB != null) {
KinesisBinderConfigurationProperties.Locks locks = this.configurationProperties.getLocks();
DynamoDbLockRegistry dynamoDbLockRegistry = new DynamoDbLockRegistry(dynamoDB, locks.getTable());
dynamoDbLockRegistry.setRefreshPeriod(locks.getRefreshPeriod());
dynamoDbLockRegistry.setHeartbeatPeriod(locks.getHeartbeatPeriod());
dynamoDbLockRegistry.setLeaseDuration(locks.getLeaseDuration());
dynamoDbLockRegistry.setPartitionKey(locks.getPartitionKey());
dynamoDbLockRegistry.setSortKeyName(locks.getSortKeyName());
dynamoDbLockRegistry.setSortKey(locks.getSortKey());
dynamoDbLockRegistry.setBillingMode(locks.getBillingMode());
dynamoDbLockRegistry.setReadCapacity(locks.getReadCapacity());
dynamoDbLockRegistry.setWriteCapacity(locks.getWriteCapacity());
return dynamoDbLockRegistry;
}
else {
return null;
}
}
@Bean
@ConditionalOnMissingBean
@ConditionalOnBean(AmazonDynamoDBAsync.class)
@ConditionalOnProperty(name = "spring.cloud.stream.kinesis.binder.kpl-kcl-enabled", havingValue = "false",
matchIfMissing = true)
public ConcurrentMetadataStore kinesisCheckpointStore(@Autowired(required = false) AmazonDynamoDBAsync dynamoDB) {
if (dynamoDB != null) {
KinesisBinderConfigurationProperties.Checkpoint checkpoint = this.configurationProperties.getCheckpoint();
DynamoDbMetadataStore kinesisCheckpointStore = new DynamoDbMetadataStore(dynamoDB, checkpoint.getTable());
kinesisCheckpointStore.setBillingMode(checkpoint.getBillingMode());
kinesisCheckpointStore.setReadCapacity(checkpoint.getReadCapacity());
kinesisCheckpointStore.setWriteCapacity(checkpoint.getWriteCapacity());
kinesisCheckpointStore.setCreateTableDelay(checkpoint.getCreateDelay());
kinesisCheckpointStore.setCreateTableRetries(checkpoint.getCreateRetries());
if (checkpoint.getTimeToLive() != null) {
kinesisCheckpointStore.setTimeToLive(checkpoint.getTimeToLive());
}
return kinesisCheckpointStore;
}
else {
return null;
}
}
@Bean
@ConditionalOnMissingBean
public AmazonDynamoDBStreams dynamoDBStreams() {
if (this.hasInputs) {
return AmazonDynamoDBStreamsClientBuilder.standard()
.withCredentials(this.awsCredentialsProvider)
.withRegion(this.region)
.build();
}
else {
return null;
}
}
@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(name = "spring.cloud.stream.kinesis.binder.kpl-kcl-enabled")
public AmazonCloudWatchAsync cloudWatch() {
if (this.hasInputs) {
return AmazonCloudWatchAsyncClientBuilder.standard()
.withCredentials(this.awsCredentialsProvider)
.withRegion(this.region)
.build();
}
else {
return null;
}
}
@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(name = "spring.cloud.stream.kinesis.binder.kpl-kcl-enabled")
public KinesisProducerConfiguration kinesisProducerConfiguration() {
KinesisProducerConfiguration kinesisProducerConfiguration = new KinesisProducerConfiguration();
kinesisProducerConfiguration.setCredentialsProvider(this.awsCredentialsProvider);
kinesisProducerConfiguration.setRegion(this.region);
return kinesisProducerConfiguration;
}
@Bean
public KinesisMessageChannelBinder kinesisMessageChannelBinder(
KinesisStreamProvisioner provisioningProvider,
AmazonKinesisAsync amazonKinesis,
KinesisExtendedBindingProperties kinesisExtendedBindingProperties,
@Autowired(required = false) ConcurrentMetadataStore kinesisCheckpointStore,
@Autowired(required = false) LockRegistry lockRegistry,
@Autowired(required = false) AmazonDynamoDB dynamoDBClient,
@Autowired(required = false) AmazonDynamoDBStreams dynamoDBStreams,
@Autowired(required = false) AmazonCloudWatch cloudWatchClient,
@Autowired(required = false) KinesisProducerConfiguration kinesisProducerConfiguration,
@Autowired(required = false) ProducerMessageHandlerCustomizer<? extends AbstractAwsMessageHandler<Void>> producerMessageHandlerCustomizer,
@Autowired(required = false) ConsumerEndpointCustomizer<? extends MessageProducerSupport> consumerEndpointCustomizer,
@Autowired List<KinesisClientLibConfiguration> kinesisClientLibConfigurations) {
KinesisMessageChannelBinder kinesisMessageChannelBinder =
new KinesisMessageChannelBinder(this.configurationProperties, provisioningProvider, amazonKinesis,
this.awsCredentialsProvider, dynamoDBClient, dynamoDBStreams, cloudWatchClient);
kinesisMessageChannelBinder.setCheckpointStore(kinesisCheckpointStore);
kinesisMessageChannelBinder.setLockRegistry(lockRegistry);
kinesisMessageChannelBinder.setExtendedBindingProperties(kinesisExtendedBindingProperties);
kinesisMessageChannelBinder.setKinesisProducerConfiguration(kinesisProducerConfiguration);
kinesisMessageChannelBinder.setProducerMessageHandlerCustomizer(producerMessageHandlerCustomizer);
kinesisMessageChannelBinder.setConsumerEndpointCustomizer(consumerEndpointCustomizer);
kinesisMessageChannelBinder.setKinesisClientLibConfigurations(kinesisClientLibConfigurations);
return kinesisMessageChannelBinder;
}
@Configuration(proxyBeanMethods = false)
@ConditionalOnClass(HealthIndicator.class)
@ConditionalOnEnabledHealthIndicator("binders")
protected static class KinesisBinderHealthIndicatorConfiguration {
@Bean
@ConditionalOnMissingBean(name = "kinesisBinderHealthIndicator")
public KinesisBinderHealthIndicator kinesisBinderHealthIndicator(
KinesisMessageChannelBinder kinesisMessageChannelBinder) {
return new KinesisBinderHealthIndicator(kinesisMessageChannelBinder);
}
}
}

305
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisBinderConfigurationProperties.java Normal file
View File

@@ -0,0 +1,305 @@
/*
* Copyright 2017-2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis.properties;
import com.amazonaws.services.dynamodbv2.model.BillingMode;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.integration.aws.lock.DynamoDbLockRegistry;
import org.springframework.integration.aws.metadata.DynamoDbMetadataStore;
/**
* The Kinesis Binder specific configuration properties.
*
* @author Peter Oates
* @author Artem Bilan
* @author Jacob Severson
* @author Sergiu Pantiru
* @author Arnaud Lecollaire
* @author Asiel Caballero
*/
@ConfigurationProperties(prefix = "spring.cloud.stream.kinesis.binder")
public class KinesisBinderConfigurationProperties {
private String[] headers = new String[] { };
private int describeStreamBackoff = 1000;
private int describeStreamRetries = 50;
private boolean autoCreateStream = true;
private boolean autoAddShards = false;
private int minShardCount = 1;
/**
* Enables the usage of Amazon KCL/KPL libraries for all message consumption and production.
*/
private boolean kplKclEnabled;
private final Checkpoint checkpoint = new Checkpoint();
private final Locks locks = new Locks();
public String[] getHeaders() {
return this.headers;
}
public void setHeaders(String... headers) {
this.headers = headers;
}
public int getDescribeStreamBackoff() {
return this.describeStreamBackoff;
}
public void setDescribeStreamBackoff(int describeStreamBackoff) {
this.describeStreamBackoff = describeStreamBackoff;
}
public int getDescribeStreamRetries() {
return this.describeStreamRetries;
}
public void setDescribeStreamRetries(int describeStreamRetries) {
this.describeStreamRetries = describeStreamRetries;
}
public boolean isAutoAddShards() {
return this.autoAddShards;
}
public void setAutoAddShards(boolean autoAddShards) {
this.autoAddShards = autoAddShards;
}
public int getMinShardCount() {
return this.minShardCount;
}
public void setMinShardCount(int minShardCount) {
this.minShardCount = minShardCount;
}
public Checkpoint getCheckpoint() {
return this.checkpoint;
}
public Locks getLocks() {
return this.locks;
}
public boolean isAutoCreateStream() {
return this.autoCreateStream;
}
public void setAutoCreateStream(boolean autoCreateStream) {
this.autoCreateStream = autoCreateStream;
}
public boolean isKplKclEnabled() {
return this.kplKclEnabled;
}
public void setKplKclEnabled(boolean kplKclEnabled) {
this.kplKclEnabled = kplKclEnabled;
}
/**
* The checkpoint DynamoDB table configuration properties.
*/
public static class Checkpoint {
private String table = DynamoDbMetadataStore.DEFAULT_TABLE_NAME;
private BillingMode billingMode = BillingMode.PAY_PER_REQUEST;
private long readCapacity = 1L;
private long writeCapacity = 1L;
private int createDelay = 1;
private int createRetries = 25;
private Integer timeToLive;
public String getTable() {
return this.table;
}
public void setTable(String table) {
this.table = table;
}
public BillingMode getBillingMode() {
return billingMode;
}
public void setBillingMode(BillingMode billingMode) {
this.billingMode = billingMode;
}
public long getReadCapacity() {
return this.readCapacity;
}
public void setReadCapacity(long readCapacity) {
this.readCapacity = readCapacity;
}
public long getWriteCapacity() {
return this.writeCapacity;
}
public void setWriteCapacity(long writeCapacity) {
this.writeCapacity = writeCapacity;
}
public int getCreateDelay() {
return this.createDelay;
}
public void setCreateDelay(int createDelay) {
this.createDelay = createDelay;
}
public int getCreateRetries() {
return this.createRetries;
}
public void setCreateRetries(int createRetries) {
this.createRetries = createRetries;
}
public Integer getTimeToLive() {
return this.timeToLive;
}
public void setTimeToLive(Integer timeToLive) {
this.timeToLive = timeToLive;
}
}
/**
* The locks DynamoDB table configuration properties.
*/
public static class Locks {
private String table = DynamoDbLockRegistry.DEFAULT_TABLE_NAME;
private BillingMode billingMode = BillingMode.PAY_PER_REQUEST;
private long readCapacity = 1L;
private long writeCapacity = 1L;
private String partitionKey = DynamoDbLockRegistry.DEFAULT_PARTITION_KEY_NAME;
private String sortKeyName = DynamoDbLockRegistry.DEFAULT_SORT_KEY_NAME;
private String sortKey = DynamoDbLockRegistry.DEFAULT_SORT_KEY;
private long refreshPeriod = DynamoDbLockRegistry.DEFAULT_REFRESH_PERIOD_MS;
private long leaseDuration = 20L;
private long heartbeatPeriod = 5L;
public String getTable() {
return this.table;
}
public void setTable(String table) {
this.table = table;
}
public BillingMode getBillingMode() {
return billingMode;
}
public void setBillingMode(BillingMode billingMode) {
this.billingMode = billingMode;
}
public long getReadCapacity() {
return this.readCapacity;
}
public void setReadCapacity(long readCapacity) {
this.readCapacity = readCapacity;
}
public long getWriteCapacity() {
return this.writeCapacity;
}
public void setWriteCapacity(long writeCapacity) {
this.writeCapacity = writeCapacity;
}
public String getPartitionKey() {
return this.partitionKey;
}
public void setPartitionKey(String partitionKey) {
this.partitionKey = partitionKey;
}
public String getSortKeyName() {
return this.sortKeyName;
}
public void setSortKeyName(String sortKeyName) {
this.sortKeyName = sortKeyName;
}
public String getSortKey() {
return this.sortKey;
}
public void setSortKey(String sortKey) {
this.sortKey = sortKey;
}
public long getRefreshPeriod() {
return this.refreshPeriod;
}
public void setRefreshPeriod(long refreshPeriod) {
this.refreshPeriod = refreshPeriod;
}
public long getLeaseDuration() {
return this.leaseDuration;
}
public void setLeaseDuration(long leaseDuration) {
this.leaseDuration = leaseDuration;
}
public long getHeartbeatPeriod() {
return this.heartbeatPeriod;
}
public void setHeartbeatPeriod(long heartbeatPeriod) {
this.heartbeatPeriod = heartbeatPeriod;
}
}
}

49
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisBindingProperties.java Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright 2017-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis.properties;
import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider;
/**
* The Kinesis-specific binding configuration properties.
*
* @author Peter Oates
* @author Artem Bilan
*/
public class KinesisBindingProperties implements BinderSpecificPropertiesProvider {
private KinesisConsumerProperties consumer = new KinesisConsumerProperties();
private KinesisProducerProperties producer = new KinesisProducerProperties();
public KinesisConsumerProperties getConsumer() {
return this.consumer;
}
public void setConsumer(KinesisConsumerProperties consumer) {
this.consumer = consumer;
}
public KinesisProducerProperties getProducer() {
return this.producer;
}
public void setProducer(KinesisProducerProperties producer) {
this.producer = producer;
}
}

150
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisConsumerProperties.java Normal file
View File

@@ -0,0 +1,150 @@
/*
* Copyright 2017-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis.properties;
import org.springframework.integration.aws.inbound.kinesis.CheckpointMode;
import org.springframework.integration.aws.inbound.kinesis.ListenerMode;
/**
* The Kinesis-specific consumer binding configuration properties.
*
* @author Peter Oates
* @author Jacob Severson
* @author Artem Bilan
* @author Arnaud Lecollaire
* @author Dmytro Danilenkov
*
*/
public class KinesisConsumerProperties {
private int startTimeout = 60000;
private ListenerMode listenerMode = ListenerMode.record;
private CheckpointMode checkpointMode = CheckpointMode.batch;
/**
* Interval, in milliseconds, between two checkpoints when checkpoint mode is periodic.
*/
private Long checkpointInterval = 5_000L;
private int recordsLimit = 10000;
private int idleBetweenPolls = 1000;
private int consumerBackoff = 1000;
private String shardIteratorType;
private boolean dynamoDbStreams;
private String shardId;
/**
* Worker identifier used to distinguish different workers/processes
* (only used when KCL is enabled).
*/
private String workerId;
public int getStartTimeout() {
return this.startTimeout;
}
public void setStartTimeout(int startTimeout) {
this.startTimeout = startTimeout;
}
public ListenerMode getListenerMode() {
return this.listenerMode;
}
public void setListenerMode(ListenerMode listenerMode) {
this.listenerMode = listenerMode;
}
public CheckpointMode getCheckpointMode() {
return this.checkpointMode;
}
public void setCheckpointMode(CheckpointMode checkpointMode) {
this.checkpointMode = checkpointMode;
}
public Long getCheckpointInterval() {
return checkpointInterval;
}
public void setCheckpointInterval(Long checkpointInterval) {
this.checkpointInterval = checkpointInterval;
}
public int getRecordsLimit() {
return this.recordsLimit;
}
public void setRecordsLimit(int recordsLimit) {
this.recordsLimit = recordsLimit;
}
public int getIdleBetweenPolls() {
return this.idleBetweenPolls;
}
public void setIdleBetweenPolls(int idleBetweenPolls) {
this.idleBetweenPolls = idleBetweenPolls;
}
public int getConsumerBackoff() {
return this.consumerBackoff;
}
public void setConsumerBackoff(int consumerBackoff) {
this.consumerBackoff = consumerBackoff;
}
public String getShardIteratorType() {
return this.shardIteratorType;
}
public void setShardIteratorType(String shardIteratorType) {
this.shardIteratorType = shardIteratorType;
}
public String getWorkerId() {
return workerId;
}
public void setWorkerId(String workerId) {
this.workerId = workerId;
}
public boolean isDynamoDbStreams() {
return this.dynamoDbStreams;
}
public void setDynamoDbStreams(boolean dynamoDbStreams) {
this.dynamoDbStreams = dynamoDbStreams;
}
public String getShardId() {
return shardId;
}
public void setShardId(String shardId) {
this.shardId = shardId;
}
}

81
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisExtendedBindingProperties.java Normal file
View File

@@ -0,0 +1,81 @@
/*
* Copyright 2017-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis.properties;
import java.util.HashMap;
import java.util.Map;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider;
import org.springframework.cloud.stream.binder.ExtendedBindingProperties;
/**
* The extended Kinesis-specific binding configuration properties.
*
* @author Peter Oates
* @author Artem Bilan
*
*/
@ConfigurationProperties("spring.cloud.stream.kinesis")
public class KinesisExtendedBindingProperties implements
ExtendedBindingProperties<KinesisConsumerProperties, KinesisProducerProperties> {
private static final String DEFAULTS_PREFIX = "spring.cloud.stream.kinesis.default";
private Map<String, KinesisBindingProperties> bindings = new HashMap<>();
public Map<String, KinesisBindingProperties> getBindings() {
return this.bindings;
}
public void setBindings(Map<String, KinesisBindingProperties> bindings) {
this.bindings = bindings;
}
@Override
public KinesisConsumerProperties getExtendedConsumerProperties(String channelName) {
if (this.bindings.containsKey(channelName)
&& this.bindings.get(channelName).getConsumer() != null) {
return this.bindings.get(channelName).getConsumer();
}
else {
return new KinesisConsumerProperties();
}
}
@Override
public KinesisProducerProperties getExtendedProducerProperties(String channelName) {
if (this.bindings.containsKey(channelName)
&& this.bindings.get(channelName).getProducer() != null) {
return this.bindings.get(channelName).getProducer();
}
else {
return new KinesisProducerProperties();
}
}
@Override
public String getDefaultsPrefix() {
return DEFAULTS_PREFIX;
}
@Override
public Class<? extends BinderSpecificPropertiesProvider> getExtendedPropertiesEntryClass() {
return KinesisBindingProperties.class;
}
}

48
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisProducerProperties.java Normal file
View File

@@ -0,0 +1,48 @@
/*
* Copyright 2017-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis.properties;
/**
* The Kinesis-specific producer binding configuration properties.
*
* @author Peter Oates
* @author Jacob Severson
*
*/
public class KinesisProducerProperties {
private boolean sync;
private long sendTimeout = 10000;
public void setSync(boolean sync) {
this.sync = sync;
}
public boolean isSync() {
return this.sync;
}
public long getSendTimeout() {
return this.sendTimeout;
}
public void setSendTimeout(long sendTimeout) {
this.sendTimeout = sendTimeout;
}
}

64
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/provisioning/KinesisConsumerDestination.java Normal file
View File

@@ -0,0 +1,64 @@
/*
* Copyright 2017-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis.provisioning;
import java.util.List;
import com.amazonaws.services.kinesis.model.Shard;
import org.springframework.cloud.stream.provisioning.ConsumerDestination;
/**
* The Kinesis-specific {@link ConsumerDestination} implementation.
*
* @author Artem Bilan
*
*/
public final class KinesisConsumerDestination implements ConsumerDestination {
private final String streamName;
private final List<Shard> shards;
private final String dlqName;
public KinesisConsumerDestination(String streamName, List<Shard> shards) {
this(streamName, shards, null);
}
public KinesisConsumerDestination(String streamName, List<Shard> shards, String dlqName) {
this.streamName = streamName;
this.shards = shards;
this.dlqName = dlqName;
}
@Override
public String getName() {
return this.streamName;
}
public List<Shard> getShards() {
return this.shards;
}
@Override
public String toString() {
return "KinesisConsumerDestination{" + "streamName='" + this.streamName + '\''
+ ", shards=" + this.shards + ", dlqName='" + this.dlqName + '\'' + '}';
}
}

62
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/provisioning/KinesisProducerDestination.java Normal file
View File

@@ -0,0 +1,62 @@
/*
* Copyright 2017-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis.provisioning;
import java.util.List;
import com.amazonaws.services.kinesis.model.Shard;
import org.springframework.cloud.stream.provisioning.ProducerDestination;
/**
* The Kinesis-specific {@link ProducerDestination} implementation.
*
* @author Artem Bilan
*
*/
public final class KinesisProducerDestination implements ProducerDestination {
private final String streamName;
private final List<Shard> shards;
KinesisProducerDestination(String streamName, List<Shard> shards) {
this.streamName = streamName;
this.shards = shards;
}
@Override
public String getName() {
return this.streamName;
}
@Override
public String getNameForPartition(int shard) {
return this.streamName;
}
public List<Shard> getShards() {
return this.shards;
}
@Override
public String toString() {
return "KinesisProducerDestination{" + "streamName='" + this.streamName + '\''
+ ", shards=" + this.shards + '}';
}
}

297
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/provisioning/KinesisStreamProvisioner.java Normal file
View File

@@ -0,0 +1,297 @@
/*
* Copyright 2017-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis.provisioning;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import com.amazonaws.services.kinesis.AmazonKinesis;
import com.amazonaws.services.kinesis.model.DescribeStreamRequest;
import com.amazonaws.services.kinesis.model.DescribeStreamResult;
import com.amazonaws.services.kinesis.model.LimitExceededException;
import com.amazonaws.services.kinesis.model.ListShardsRequest;
import com.amazonaws.services.kinesis.model.ListShardsResult;
import com.amazonaws.services.kinesis.model.ResourceNotFoundException;
import com.amazonaws.services.kinesis.model.ScalingType;
import com.amazonaws.services.kinesis.model.Shard;
import com.amazonaws.services.kinesis.model.StreamDescription;
import com.amazonaws.services.kinesis.model.StreamStatus;
import com.amazonaws.services.kinesis.model.UpdateShardCountRequest;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
import org.springframework.cloud.stream.binder.HeaderMode;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisBinderConfigurationProperties;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisConsumerProperties;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisProducerProperties;
import org.springframework.cloud.stream.provisioning.ConsumerDestination;
import org.springframework.cloud.stream.provisioning.ProducerDestination;
import org.springframework.cloud.stream.provisioning.ProvisioningException;
import org.springframework.cloud.stream.provisioning.ProvisioningProvider;
import org.springframework.util.Assert;
/**
* The {@link ProvisioningProvider} implementation for Amazon Kinesis.
*
* @author Peter Oates
* @author Artem Bilan
* @author Jacob Severson
* @author Sergiu Pantiru
* @author Matthias Wesolowski
*/
public class KinesisStreamProvisioner implements
ProvisioningProvider<ExtendedConsumerProperties<KinesisConsumerProperties>,
ExtendedProducerProperties<KinesisProducerProperties>> {
private static final Log logger = LogFactory.getLog(KinesisStreamProvisioner.class);
private final AmazonKinesis amazonKinesis;
private final KinesisBinderConfigurationProperties configurationProperties;
public KinesisStreamProvisioner(AmazonKinesis amazonKinesis,
KinesisBinderConfigurationProperties kinesisBinderConfigurationProperties) {
Assert.notNull(amazonKinesis, "'amazonKinesis' must not be null");
Assert.notNull(kinesisBinderConfigurationProperties,
"'kinesisBinderConfigurationProperties' must not be null");
this.amazonKinesis = amazonKinesis;
this.configurationProperties = kinesisBinderConfigurationProperties;
}
@Override
public ProducerDestination provisionProducerDestination(String name,
ExtendedProducerProperties<KinesisProducerProperties> properties)
throws ProvisioningException {
if (logger.isInfoEnabled()) {
logger.info("Using Kinesis stream for outbound: " + name);
}
if (properties.getHeaderMode() == null) {
properties.setHeaderMode(HeaderMode.embeddedHeaders);
}
return new KinesisProducerDestination(name,
createOrUpdate(name, properties.getPartitionCount()));
}
@Override
public ConsumerDestination provisionConsumerDestination(String name, String group,
ExtendedConsumerProperties<KinesisConsumerProperties> properties)
throws ProvisioningException {
if (properties.getExtension().isDynamoDbStreams()) {
if (logger.isInfoEnabled()) {
logger.info("Using DynamoDB table in DynamoDB Streams support for inbound: " + name);
}
return new KinesisConsumerDestination(name, Collections.emptyList());
}
if (logger.isInfoEnabled()) {
logger.info("Using Kinesis stream for inbound: " + name);
}
if (properties.getHeaderMode() == null) {
properties.setHeaderMode(HeaderMode.embeddedHeaders);
}
int shardCount = properties.getInstanceCount() * properties.getConcurrency();
return new KinesisConsumerDestination(name, createOrUpdate(name, shardCount));
}
private List<Shard> getShardList(String stream) {
return this.getShardList(stream, 0);
}
private List<Shard> getShardList(String stream, int retryCount) {
List<Shard> shardList = new ArrayList<>();
if (retryCount++ > configurationProperties.getDescribeStreamRetries()) {
ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(
"The stream [" + stream + "] isn't ACTIVE or doesn't exist.");
resourceNotFoundException.setServiceName("Kinesis");
throw new ProvisioningException(
"Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error",
resourceNotFoundException);
}
ListShardsRequest listShardsRequest = new ListShardsRequest().withStreamName(stream);
try {
ListShardsResult listShardsResult = amazonKinesis.listShards(listShardsRequest);
shardList.addAll(listShardsResult.getShards());
}
catch (LimitExceededException limitExceededException) {
logger.info("Got LimitExceededException when describing stream [" + stream + "]. " + "Backing off for ["
+ this.configurationProperties.getDescribeStreamBackoff() + "] millis.");
try {
Thread.sleep(this.configurationProperties.getDescribeStreamBackoff());
getShardList(stream, retryCount);
}
catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new ProvisioningException(
"The [describeStream] thread for the stream [" + stream + "] has been interrupted.", ex);
}
}
return shardList;
}
private List<Shard> createOrUpdate(String stream, int shards) {
List<Shard> shardList = new ArrayList<>();
try {
shardList = getShardList(stream);
}
catch (ResourceNotFoundException ex) {
if (!this.configurationProperties.isAutoCreateStream()) {
throw new ProvisioningException(
"The stream [" + stream + "] was not found and auto creation is disabled.", ex);
}
if (logger.isInfoEnabled()) {
logger.info("Stream '" + stream + "' not found. Create one...");
}
this.amazonKinesis.createStream(stream, Math.max(this.configurationProperties.getMinShardCount(), shards));
waitForStreamToBecomeActive(stream);
}
int effectiveShardCount = Math.max(this.configurationProperties.getMinShardCount(), shards);
if ((shardList.size() < effectiveShardCount)
&& this.configurationProperties.isAutoAddShards()) {
return updateShardCount(stream, shardList.size(), effectiveShardCount);
}
return shardList;
}
private void waitForStreamToBecomeActive(String streamName) {
int describeStreamRetries = 0;
while (true) {
try {
DescribeStreamResult describeStreamResult = this.amazonKinesis.describeStream(streamName);
if (describeStreamResult != null &&
StreamStatus.ACTIVE.name().equals(describeStreamResult.getStreamDescription().getStreamStatus())) {
return;
}
else if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) {
ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(
"The stream [" + streamName + "] isn't ACTIVE or doesn't exist.");
resourceNotFoundException.setServiceName("Kinesis");
throw new ProvisioningException(
"Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error",
resourceNotFoundException);
}
}
catch (LimitExceededException ex) {
logger.info("Got LimitExceededException when describing stream [" + streamName + "]. " +
"Backing off for [" + this.configurationProperties.getDescribeStreamBackoff() + "] millis.");
}
try {
Thread.sleep(this.configurationProperties.getDescribeStreamBackoff());
}
catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new ProvisioningException(
"The [describeStream] thread for the stream [" + streamName + "] has been interrupted.", ex);
}
}
}
private List<Shard> updateShardCount(String streamName, int shardCount, int targetCount) {
if (logger.isInfoEnabled()) {
logger.info("Stream [" + streamName + "] has [" + shardCount
+ "] shards compared to a target configuration of [" + targetCount
+ "], creating shards...");
}
UpdateShardCountRequest updateShardCountRequest = new UpdateShardCountRequest()
.withStreamName(streamName).withTargetShardCount(targetCount)
.withScalingType(ScalingType.UNIFORM_SCALING);
this.amazonKinesis.updateShardCount(updateShardCountRequest);
// Wait for stream to become active again after resharding
List<Shard> shardList = new ArrayList<>();
int describeStreamRetries = 0;
String exclusiveStartShardId = null;
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
.withStreamName(streamName);
while (true) {
DescribeStreamResult describeStreamResult = null;
try {
describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId);
describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest);
StreamDescription streamDescription = describeStreamResult.getStreamDescription();
if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) {
shardList.addAll(streamDescription.getShards());
if (streamDescription.getHasMoreShards()) {
exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId();
}
else {
break;
}
}
}
catch (LimitExceededException ex) {
logger.info("Got LimitExceededException when describing stream [" + streamName + "]. "
+ "Backing off for [" + this.configurationProperties.getDescribeStreamBackoff() + "] millis.");
}
if (describeStreamResult == null || !StreamStatus.ACTIVE.toString().equals(
describeStreamResult.getStreamDescription().getStreamStatus())) {
if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) {
ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(
"The stream [" + streamName + "] isn't ACTIVE or doesn't exist.");
resourceNotFoundException.setServiceName("Kinesis");
throw new ProvisioningException(
"Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error",
resourceNotFoundException);
}
try {
Thread.sleep(this.configurationProperties.getDescribeStreamBackoff());
}
catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new ProvisioningException(
"The [describeStream] thread for the stream [" + streamName + "] has been interrupted.", ex);
}
}
}
return shardList;
}
}

1
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/main/resources/META-INF/spring.binders Normal file
View File

@@ -0,0 +1 @@
kinesis: org.springframework.cloud.stream.binder.kinesis.config.KinesisBinderConfiguration

181
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/test/java/org/springframework/cloud/stream/binder/kinesis/KinesisBinderFunctionalTests.java Normal file
View File

@@ -0,0 +1,181 @@
/*
* Copyright 2021-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import com.amazonaws.SDKGlobalConfiguration;
import com.amazonaws.services.kinesis.AmazonKinesisAsync;
import com.amazonaws.services.kinesis.model.PutRecordsRequest;
import com.amazonaws.services.kinesis.model.PutRecordsRequestEntry;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.awspring.cloud.autoconfigure.context.ContextResourceLoaderAutoConfiguration;
import io.awspring.cloud.autoconfigure.context.ContextStackAutoConfiguration;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.integration.aws.support.AwsHeaders;
import org.springframework.integration.metadata.ConcurrentMetadataStore;
import org.springframework.integration.metadata.SimpleMetadataStore;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.integration.support.json.JacksonJsonUtils;
import org.springframework.integration.support.locks.DefaultLockRegistry;
import org.springframework.integration.support.locks.LockRegistry;
import org.springframework.messaging.Message;
import org.springframework.messaging.support.GenericMessage;
import org.springframework.test.annotation.DirtiesContext;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Artem Bilan
*/
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE,
properties = {
"spring.cloud.stream.bindings.eventConsumerBatchProcessingWithHeaders-in-0.destination=" + KinesisBinderFunctionalTests.KINESIS_STREAM,
"spring.cloud.stream.kinesis.bindings.eventConsumerBatchProcessingWithHeaders-in-0.consumer.idleBetweenPolls = 1",
"spring.cloud.stream.kinesis.bindings.eventConsumerBatchProcessingWithHeaders-in-0.consumer.listenerMode = batch",
"spring.cloud.stream.kinesis.bindings.eventConsumerBatchProcessingWithHeaders-in-0.consumer.checkpointMode = manual",
"spring.cloud.stream.kinesis.binder.headers = event.eventType",
"spring.cloud.stream.kinesis.binder.autoAddShards = true",
"cloud.aws.region.static=eu-west-2" })
@DirtiesContext
public class KinesisBinderFunctionalTests implements LocalstackContainerTest {
static final String KINESIS_STREAM = "test_stream";
private static AmazonKinesisAsync AMAZON_KINESIS;
@Autowired
private ObjectMapper objectMapper;
@Autowired
private CountDownLatch messageBarrier;
@Autowired
private AtomicReference<Message<List<?>>> messageHolder;
@BeforeAll
static void setup() {
AMAZON_KINESIS = LocalstackContainerTest.kinesisClient();
System.setProperty(SDKGlobalConfiguration.AWS_CBOR_DISABLE_SYSTEM_PROPERTY, "true");
}
@Test
void testKinesisFunction() throws JsonProcessingException, InterruptedException {
PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
putRecordsRequest.setStreamName(KINESIS_STREAM);
List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
for (int i = 0; i < 10; i++) {
Message<String> eventMessages =
MessageBuilder.withPayload("Message" + i)
.setHeader("event.eventType", "createEvent")
.build();
PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
byte[] jsonInput = objectMapper.writeValueAsBytes(eventMessages);
putRecordsRequestEntry.setData(ByteBuffer.wrap(jsonInput));
putRecordsRequestEntry.setPartitionKey("1");
putRecordsRequestEntryList.add(putRecordsRequestEntry);
}
putRecordsRequest.setRecords(putRecordsRequestEntryList);
AMAZON_KINESIS.putRecords(putRecordsRequest);
assertThat(this.messageBarrier.await(10, TimeUnit.SECONDS)).isTrue();
Message<List<?>> message = this.messageHolder.get();
assertThat(message.getHeaders())
.containsKeys(AwsHeaders.CHECKPOINTER,
AwsHeaders.SHARD,
AwsHeaders.RECEIVED_PARTITION_KEY,
AwsHeaders.RECEIVED_STREAM,
AwsHeaders.RECEIVED_SEQUENCE_NUMBER)
.doesNotContainKeys(AwsHeaders.STREAM, AwsHeaders.PARTITION_KEY);
List<?> payload = message.getPayload();
assertThat(payload).hasSize(10);
Object item = payload.get(0);
assertThat(item).isInstanceOf(GenericMessage.class);
Message<?> messageFromBatch = (Message<?>) item;
assertThat(messageFromBatch.getPayload()).isEqualTo("Message0");
assertThat(messageFromBatch.getHeaders())
.containsEntry("event.eventType", "createEvent");
}
@Configuration
@EnableAutoConfiguration(exclude = {
ContextResourceLoaderAutoConfiguration.class,
ContextStackAutoConfiguration.class })
static class TestConfiguration {
@Bean(destroyMethod = "")
public AmazonKinesisAsync amazonKinesis() {
return AMAZON_KINESIS;
}
@Bean
public LockRegistry lockRegistry() {
return new DefaultLockRegistry();
}
@Bean
public ConcurrentMetadataStore checkpointStore() {
return new SimpleMetadataStore();
}
@Bean
public ObjectMapper objectMapper() {
return JacksonJsonUtils.messagingAwareMapper();
}
@Bean
public AtomicReference<Message<List<?>>> messageHolder() {
return new AtomicReference<>();
}
@Bean
public CountDownLatch messageBarrier() {
return new CountDownLatch(1);
}
@Bean
public Consumer<Message<List<?>>> eventConsumerBatchProcessingWithHeaders() {
return eventMessages -> {
messageHolder().set(eventMessages);
messageBarrier().countDown();
};
}
}
}

443
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/test/java/org/springframework/cloud/stream/binder/kinesis/KinesisBinderTests.java Normal file
View File

@@ -0,0 +1,443 @@
/*
* Copyright 2017-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import com.amazonaws.SDKGlobalConfiguration;
import com.amazonaws.handlers.AsyncHandler;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
import com.amazonaws.services.kinesis.AmazonKinesisAsync;
import com.amazonaws.services.kinesis.model.DescribeStreamRequest;
import com.amazonaws.services.kinesis.model.DescribeStreamResult;
import com.amazonaws.services.kinesis.model.PutRecordRequest;
import com.amazonaws.services.kinesis.model.PutRecordResult;
import com.amazonaws.services.kinesis.model.Record;
import com.amazonaws.services.kinesis.model.Shard;
import com.amazonaws.services.kinesis.model.ShardIteratorType;
import com.amazonaws.services.kinesis.model.StreamDescription;
import com.amazonaws.services.kinesis.model.StreamStatus;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.mockito.BDDMockito;
import org.mockito.stubbing.Answer;
import org.springframework.beans.DirectFieldAccessor;
import org.springframework.cloud.stream.binder.Binding;
import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
import org.springframework.cloud.stream.binder.PartitionCapableBinderTests;
import org.springframework.cloud.stream.binder.Spy;
import org.springframework.cloud.stream.binder.TestUtils;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisBinderConfigurationProperties;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisConsumerProperties;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisProducerProperties;
import org.springframework.cloud.stream.config.BindingProperties;
import org.springframework.context.ApplicationContext;
import org.springframework.expression.common.LiteralExpression;
import org.springframework.integration.aws.inbound.kinesis.KinesisShardOffset;
import org.springframework.integration.aws.inbound.kinesis.ListenerMode;
import org.springframework.integration.aws.support.AwsRequestFailureException;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.channel.QueueChannel;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.MessageHeaders;
import org.springframework.messaging.SubscribableChannel;
import org.springframework.messaging.support.ErrorMessage;
import org.springframework.messaging.support.GenericMessage;
import org.springframework.util.MimeTypeUtils;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
/**
* The tests for Kinesis Binder.
*
* @author Artem Bilan
* @author Jacob Severson
* @author Arnaud Lecollaire
*/
public class KinesisBinderTests extends
PartitionCapableBinderTests<KinesisTestBinder, ExtendedConsumerProperties<KinesisConsumerProperties>,
ExtendedProducerProperties<KinesisProducerProperties>>
implements LocalstackContainerTest {
private static final String CLASS_UNDER_TEST_NAME = KinesisBinderTests.class
.getSimpleName();
private static AmazonKinesisAsync AMAZON_KINESIS;
private static AmazonDynamoDBAsync DYNAMO_DB;
public KinesisBinderTests() {
this.timeoutMultiplier = 10D;
}
@BeforeAll
public static void setup() {
AMAZON_KINESIS = LocalstackContainerTest.kinesisClient();
DYNAMO_DB = LocalstackContainerTest.dynamoDbClient();
System.setProperty(SDKGlobalConfiguration.AWS_CBOR_DISABLE_SYSTEM_PROPERTY, "true");
}
@Test
@Override
public void testClean(TestInfo testInfo) {
}
@Test
@Override
public void testPartitionedModuleSpEL(TestInfo testInfo) {
}
@Test
public void testAutoCreateStreamForNonExistingStream() throws Exception {
KinesisTestBinder binder = getBinder();
DirectChannel output = createBindableChannel("output", new BindingProperties());
ExtendedConsumerProperties<KinesisConsumerProperties> consumerProperties = createConsumerProperties();
Date testDate = new Date();
consumerProperties.getExtension().setShardIteratorType(
ShardIteratorType.AT_TIMESTAMP.name() + ":" + testDate.getTime());
String testStreamName = "nonexisting" + System.currentTimeMillis();
Binding<?> binding = binder.bindConsumer(testStreamName, "test", output,
consumerProperties);
binding.unbind();
DescribeStreamResult streamResult = AMAZON_KINESIS.describeStream(testStreamName);
String createdStreamName = streamResult.getStreamDescription().getStreamName();
int createdShards = streamResult.getStreamDescription().getShards().size();
String createdStreamStatus = streamResult.getStreamDescription()
.getStreamStatus();
assertThat(createdStreamName).isEqualTo(testStreamName);
assertThat(createdShards).isEqualTo(consumerProperties.getInstanceCount()
* consumerProperties.getConcurrency());
assertThat(createdStreamStatus).isEqualTo(StreamStatus.ACTIVE.toString());
KinesisShardOffset shardOffset = TestUtils.getPropertyValue(binding,
"lifecycle.streamInitialSequence", KinesisShardOffset.class);
assertThat(shardOffset.getIteratorType())
.isEqualTo(ShardIteratorType.AT_TIMESTAMP);
assertThat(shardOffset.getTimestamp()).isEqualTo(testDate);
}
@Test
@Override
@SuppressWarnings("unchecked")
public void testAnonymousGroup(TestInfo testInfo) throws Exception {
KinesisTestBinder binder = getBinder();
ExtendedProducerProperties<KinesisProducerProperties> producerProperties = createProducerProperties();
DirectChannel output = createBindableChannel("output",
createProducerBindingProperties(producerProperties));
Binding<MessageChannel> producerBinding = binder.bindProducer(
String.format("defaultGroup%s0", getDestinationNameDelimiter()), output,
producerProperties);
ExtendedConsumerProperties<KinesisConsumerProperties> consumerProperties = createConsumerProperties();
consumerProperties.setConcurrency(2);
consumerProperties.setInstanceCount(3);
consumerProperties.setInstanceIndex(0);
QueueChannel input1 = new QueueChannel();
Binding<MessageChannel> binding1 = binder.bindConsumer(
String.format("defaultGroup%s0", getDestinationNameDelimiter()), null,
input1, consumerProperties);
consumerProperties.setInstanceIndex(1);
QueueChannel input2 = new QueueChannel();
Binding<MessageChannel> binding2 = binder.bindConsumer(
String.format("defaultGroup%s0", getDestinationNameDelimiter()), null,
input2, consumerProperties);
String testPayload1 = "foo-" + UUID.randomUUID().toString();
output.send(MessageBuilder.withPayload(testPayload1)
.setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN)
.build());
Message<byte[]> receivedMessage1 = (Message<byte[]>) receive(input1);
assertThat(receivedMessage1).isNotNull();
assertThat(new String(receivedMessage1.getPayload())).isEqualTo(testPayload1);
Message<byte[]> receivedMessage2 = (Message<byte[]>) receive(input2);
assertThat(receivedMessage2).isNotNull();
assertThat(new String(receivedMessage2.getPayload())).isEqualTo(testPayload1);
binding2.unbind();
String testPayload2 = "foo-" + UUID.randomUUID().toString();
output.send(MessageBuilder.withPayload(testPayload2)
.setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN)
.build());
binding2 = binder.bindConsumer(
String.format("defaultGroup%s0", getDestinationNameDelimiter()), null,
input2, consumerProperties);
String testPayload3 = "foo-" + UUID.randomUUID().toString();
output.send(MessageBuilder.withPayload(testPayload3)
.setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN)
.build());
receivedMessage1 = (Message<byte[]>) receive(input1);
assertThat(receivedMessage1).isNotNull();
assertThat(new String(receivedMessage1.getPayload())).isEqualTo(testPayload2);
receivedMessage1 = (Message<byte[]>) receive(input1);
assertThat(receivedMessage1).isNotNull();
assertThat(new String(receivedMessage1.getPayload())).isNotNull();
receivedMessage2 = (Message<byte[]>) receive(input2);
assertThat(receivedMessage2).isNotNull();
assertThat(new String(receivedMessage2.getPayload())).isEqualTo(testPayload1);
receivedMessage2 = (Message<byte[]>) receive(input2);
assertThat(receivedMessage2).isNotNull();
assertThat(new String(receivedMessage2.getPayload())).isEqualTo(testPayload2);
receivedMessage2 = (Message<byte[]>) receive(input2);
assertThat(receivedMessage2).isNotNull();
assertThat(new String(receivedMessage2.getPayload())).isEqualTo(testPayload3);
producerBinding.unbind();
binding1.unbind();
binding2.unbind();
}
@Test
@SuppressWarnings("unchecked")
public void testProducerErrorChannel() throws Exception {
KinesisTestBinder binder = getBinder();
final RuntimeException putRecordException = new RuntimeException(
"putRecordRequestEx");
final AtomicReference<Object> sent = new AtomicReference<>();
AmazonKinesisAsync amazonKinesisMock = mock(AmazonKinesisAsync.class);
BDDMockito
.given(amazonKinesisMock.putRecordAsync(any(PutRecordRequest.class),
any(AsyncHandler.class)))
.willAnswer((Answer<Future<PutRecordResult>>) (invocation) -> {
PutRecordRequest request = invocation.getArgument(0);
sent.set(request.getData());
AsyncHandler<?, ?> handler = invocation.getArgument(1);
handler.onError(putRecordException);
return mock(Future.class);
});
new DirectFieldAccessor(binder.getBinder()).setPropertyValue("amazonKinesis",
amazonKinesisMock);
ExtendedProducerProperties<KinesisProducerProperties> producerProps = createProducerProperties();
producerProps.setErrorChannelEnabled(true);
DirectChannel moduleOutputChannel = createBindableChannel("output",
createProducerBindingProperties(producerProps));
Binding<MessageChannel> producerBinding = binder.bindProducer("ec.0",
moduleOutputChannel, producerProps);
ApplicationContext applicationContext = TestUtils.getPropertyValue(
binder.getBinder(), "applicationContext", ApplicationContext.class);
SubscribableChannel ec = applicationContext.getBean("ec.0.errors",
SubscribableChannel.class);
final AtomicReference<Message<?>> errorMessage = new AtomicReference<>();
final CountDownLatch latch = new CountDownLatch(1);
ec.subscribe((message) -> {
errorMessage.set(message);
latch.countDown();
});
String messagePayload = "oops";
moduleOutputChannel.send(new GenericMessage<>(messagePayload.getBytes()));
assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
assertThat(errorMessage.get()).isInstanceOf(ErrorMessage.class);
assertThat(errorMessage.get().getPayload())
.isInstanceOf(AwsRequestFailureException.class);
AwsRequestFailureException exception = (AwsRequestFailureException) errorMessage
.get().getPayload();
assertThat(exception.getCause()).isSameAs(putRecordException);
assertThat(((PutRecordRequest) exception.getRequest()).getData())
.isSameAs(sent.get());
producerBinding.unbind();
}
@Test
@SuppressWarnings("unchecked")
public void testBatchListener() throws Exception {
KinesisTestBinder binder = getBinder();
ExtendedProducerProperties<KinesisProducerProperties> producerProperties = createProducerProperties();
DirectChannel output = createBindableChannel("output",
createProducerBindingProperties(producerProperties));
Binding<MessageChannel> outputBinding = binder.bindProducer("testBatchListener",
output, producerProperties);
for (int i = 0; i < 3; i++) {
output.send(new GenericMessage<>(i));
}
ExtendedConsumerProperties<KinesisConsumerProperties> consumerProperties = createConsumerProperties();
consumerProperties.getExtension().setListenerMode(ListenerMode.batch);
consumerProperties.setUseNativeDecoding(true);
QueueChannel input = new QueueChannel();
Binding<MessageChannel> inputBinding = binder.bindConsumer("testBatchListener",
null, input, consumerProperties);
Message<List<?>> receivedMessage = (Message<List<?>>) receive(input);
assertThat(receivedMessage).isNotNull();
assertThat(receivedMessage.getPayload().size()).isEqualTo(3);
receivedMessage.getPayload().forEach((r) -> {
assertThat(r).isInstanceOf(Record.class);
});
outputBinding.unbind();
inputBinding.unbind();
}
@Test
@Disabled("Localstack doesn't support updateShardCount. Test only against real AWS Kinesis")
public void testPartitionCountIncreasedIfAutoAddPartitionsSet() throws Exception {
KinesisBinderConfigurationProperties configurationProperties = new KinesisBinderConfigurationProperties();
String stream = "existing" + System.currentTimeMillis();
AMAZON_KINESIS.createStream(stream, 1);
List<Shard> shards = describeStream(stream);
assertThat(shards.size()).isEqualTo(1);
configurationProperties.setMinShardCount(6);
configurationProperties.setAutoAddShards(true);
KinesisTestBinder binder = getBinder(configurationProperties);
ExtendedConsumerProperties<KinesisConsumerProperties> consumerProperties = createConsumerProperties();
DirectChannel output = createBindableChannel("output", new BindingProperties());
Binding<?> binding = binder.bindConsumer(stream, "test", output, consumerProperties);
binding.unbind();
shards = describeStream(stream);
assertThat(shards.size()).isEqualTo(6);
}
private List<Shard> describeStream(String stream) {
String exclusiveStartShardId = null;
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
.withStreamName(stream);
List<Shard> shardList = new ArrayList<>();
while (true) {
DescribeStreamResult describeStreamResult;
describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId);
describeStreamResult = AMAZON_KINESIS.describeStream(describeStreamRequest);
StreamDescription streamDescription = describeStreamResult
.getStreamDescription();
if (StreamStatus.ACTIVE.toString()
.equals(streamDescription.getStreamStatus())) {
shardList.addAll(streamDescription.getShards());
if (streamDescription.getHasMoreShards()) {
exclusiveStartShardId = shardList.get(shardList.size() - 1)
.getShardId();
continue;
}
else {
return shardList;
}
}
try {
Thread.sleep(100);
}
catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new IllegalStateException(ex);
}
}
}
@Override
protected boolean usesExplicitRouting() {
return false;
}
@Override
protected String getClassUnderTestName() {
return CLASS_UNDER_TEST_NAME;
}
@Override
protected KinesisTestBinder getBinder() {
return getBinder(new KinesisBinderConfigurationProperties());
}
private KinesisTestBinder getBinder(
KinesisBinderConfigurationProperties kinesisBinderConfigurationProperties) {
if (this.testBinder == null) {
this.testBinder = new KinesisTestBinder(AMAZON_KINESIS, DYNAMO_DB, kinesisBinderConfigurationProperties);
this.timeoutMultiplier = 20;
}
return this.testBinder;
}
@Override
protected ExtendedConsumerProperties<KinesisConsumerProperties> createConsumerProperties() {
ExtendedConsumerProperties<KinesisConsumerProperties> kinesisConsumerProperties = new ExtendedConsumerProperties<>(
new KinesisConsumerProperties());
// set the default values that would normally be propagated by Spring Cloud Stream
kinesisConsumerProperties.setInstanceCount(1);
kinesisConsumerProperties.setInstanceIndex(0);
kinesisConsumerProperties.getExtension().setShardIteratorType(ShardIteratorType.TRIM_HORIZON.name());
return kinesisConsumerProperties;
}
private ExtendedProducerProperties<KinesisProducerProperties> createProducerProperties() {
return this.createProducerProperties(null);
}
@Override
protected ExtendedProducerProperties<KinesisProducerProperties> createProducerProperties(TestInfo testInto) {
ExtendedProducerProperties<KinesisProducerProperties> producerProperties = new ExtendedProducerProperties<>(
new KinesisProducerProperties());
producerProperties.setPartitionKeyExpression(new LiteralExpression("1"));
producerProperties.getExtension().setSync(true);
return producerProperties;
}
@Override
public Spy spyOn(String name) {
throw new UnsupportedOperationException("'spyOn' is not used by Kinesis tests");
}
}

173
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/test/java/org/springframework/cloud/stream/binder/kinesis/KinesisTestBinder.java Normal file
View File

@@ -0,0 +1,173 @@
/*
* Copyright 2017-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis;
import java.util.List;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
import com.amazonaws.services.kinesis.AmazonKinesisAsync;
import com.amazonaws.services.kinesis.model.ListStreamsRequest;
import com.amazonaws.services.kinesis.model.ListStreamsResult;
import com.amazonaws.services.kinesis.model.ResourceNotFoundException;
import org.springframework.beans.DirectFieldAccessor;
import org.springframework.cloud.stream.binder.AbstractTestBinder;
import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
import org.springframework.cloud.stream.binder.PartitionTestSupport;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisBinderConfigurationProperties;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisConsumerProperties;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisProducerProperties;
import org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner;
import org.springframework.cloud.stream.provisioning.ConsumerDestination;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.support.GenericApplicationContext;
import org.springframework.integration.config.EnableIntegration;
import org.springframework.integration.core.MessageProducer;
/**
* An {@link AbstractTestBinder} implementation for the {@link KinesisMessageChannelBinder}.
*
* @author Artem Bilan
* @author Arnaud Lecollaire
*/
public class KinesisTestBinder extends
AbstractTestBinder<KinesisMessageChannelBinder, ExtendedConsumerProperties<KinesisConsumerProperties>,
ExtendedProducerProperties<KinesisProducerProperties>> {
private final AmazonKinesisAsync amazonKinesis;
private final GenericApplicationContext applicationContext;
public KinesisTestBinder(AmazonKinesisAsync amazonKinesis, AmazonDynamoDBAsync dynamoDbClient,
KinesisBinderConfigurationProperties kinesisBinderConfigurationProperties) {
this.applicationContext = new AnnotationConfigApplicationContext(Config.class);
this.amazonKinesis = amazonKinesis;
KinesisStreamProvisioner provisioningProvider = new KinesisStreamProvisioner(
amazonKinesis, kinesisBinderConfigurationProperties);
KinesisMessageChannelBinder binder = new TestKinesisMessageChannelBinder(
amazonKinesis, dynamoDbClient, kinesisBinderConfigurationProperties,
provisioningProvider);
binder.setApplicationContext(this.applicationContext);
setBinder(binder);
}
public GenericApplicationContext getApplicationContext() {
return this.applicationContext;
}
@Override
public void cleanup() {
ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
ListStreamsResult listStreamsResult = this.amazonKinesis
.listStreams(listStreamsRequest);
List<String> streamNames = listStreamsResult.getStreamNames();
while (listStreamsResult.getHasMoreStreams()) {
if (streamNames.size() > 0) {
listStreamsRequest.setExclusiveStartStreamName(
streamNames.get(streamNames.size() - 1));
}
listStreamsResult = this.amazonKinesis.listStreams(listStreamsRequest);
streamNames.addAll(listStreamsResult.getStreamNames());
}
for (String stream : streamNames) {
this.amazonKinesis.deleteStream(stream);
while (true) {
try {
this.amazonKinesis.describeStream(stream);
try {
Thread.sleep(100);
}
catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new IllegalStateException(ex);
}
}
catch (ResourceNotFoundException ex) {
break;
}
}
}
}
/**
* Test configuration.
*/
@Configuration
@EnableIntegration
static class Config {
@Bean
public PartitionTestSupport partitionSupport() {
return new PartitionTestSupport();
}
}
private static class TestKinesisMessageChannelBinder
extends KinesisMessageChannelBinder {
TestKinesisMessageChannelBinder(AmazonKinesisAsync amazonKinesis,
AmazonDynamoDBAsync dynamoDbClient,
KinesisBinderConfigurationProperties kinesisBinderConfigurationProperties,
KinesisStreamProvisioner provisioningProvider) {
super(kinesisBinderConfigurationProperties, provisioningProvider, amazonKinesis,
new AWSStaticCredentialsProvider(new BasicAWSCredentials("", "")), dynamoDbClient, null, null);
}
/*
* Some tests use multiple instance indexes for the same topic; we need to make
* the error infrastructure beans unique.
*/
@Override
protected String errorsBaseName(ConsumerDestination destination, String group,
ExtendedConsumerProperties<KinesisConsumerProperties> consumerProperties) {
return super.errorsBaseName(destination, group, consumerProperties) + "-"
+ consumerProperties.getInstanceIndex();
}
@Override
protected MessageProducer createConsumerEndpoint(ConsumerDestination destination,
String group,
ExtendedConsumerProperties<KinesisConsumerProperties> properties) {
MessageProducer messageProducer = super.createConsumerEndpoint(destination,
group, properties);
DirectFieldAccessor dfa = new DirectFieldAccessor(messageProducer);
dfa.setPropertyValue("describeStreamBackoff", 10);
dfa.setPropertyValue("consumerBackoff", 10);
dfa.setPropertyValue("idleBetweenPolls", 1);
return messageProducer;
}
}
}

73
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/test/java/org/springframework/cloud/stream/binder/kinesis/LocalstackContainerTest.java Normal file
View File

@@ -0,0 +1,73 @@
/*
* Copyright 2022-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.cloudwatch.AmazonCloudWatch;
import com.amazonaws.services.cloudwatch.AmazonCloudWatchClientBuilder;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsyncClientBuilder;
import com.amazonaws.services.kinesis.AmazonKinesisAsync;
import com.amazonaws.services.kinesis.AmazonKinesisAsyncClientBuilder;
import org.junit.jupiter.api.condition.DisabledOnOs;
import org.junit.jupiter.api.condition.OS;
import org.testcontainers.containers.localstack.LocalStackContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
import org.springframework.integration.test.util.TestUtils;
/**
*
* @author Artem Bilan
*
* @since 4.0
*/
@Testcontainers(disabledWithoutDocker = true)
@DisabledOnOs(OS.MAC)
public interface LocalstackContainerTest {
@Container
LocalStackContainer localStack =
new LocalStackContainer(DockerImageName.parse(TestUtils.dockerRegistryFromEnv() + "localstack/localstack"))
.withServices(
LocalStackContainer.Service.DYNAMODB,
LocalStackContainer.Service.KINESIS,
LocalStackContainer.Service.CLOUDWATCH);
static AmazonDynamoDBAsync dynamoDbClient() {
return applyAwsClientOptions(AmazonDynamoDBAsyncClientBuilder.standard(), LocalStackContainer.Service.DYNAMODB);
}
static AmazonKinesisAsync kinesisClient() {
return applyAwsClientOptions(AmazonKinesisAsyncClientBuilder.standard(), LocalStackContainer.Service.KINESIS);
}
static AmazonCloudWatch cloudWatchClient() {
return applyAwsClientOptions(AmazonCloudWatchClientBuilder.standard(), LocalStackContainer.Service.CLOUDWATCH);
}
private static <B extends AwsClientBuilder<B, T>, T> T applyAwsClientOptions(B clientBuilder,
LocalStackContainer.Service serviceToBuild) {
return clientBuilder.withEndpointConfiguration(localStack.getEndpointConfiguration(serviceToBuild))
.withCredentials(localStack.getDefaultCredentialsProvider())
.build();
}
}

200
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/test/java/org/springframework/cloud/stream/binder/kinesis/adapter/SpringDynamoDBAdapterClientTests.java Normal file
View File

@@ -0,0 +1,200 @@
/*
* Copyright 2017-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis.adapter;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.dynamodbv2.AbstractAmazonDynamoDBStreams;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBStreams;
import com.amazonaws.services.dynamodbv2.model.AmazonDynamoDBException;
import com.amazonaws.services.dynamodbv2.model.DescribeStreamRequest;
import com.amazonaws.services.dynamodbv2.model.DescribeStreamResult;
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
import com.amazonaws.services.dynamodbv2.model.KeyType;
import com.amazonaws.services.dynamodbv2.model.SequenceNumberRange;
import com.amazonaws.services.dynamodbv2.model.Shard;
import com.amazonaws.services.dynamodbv2.model.StreamDescription;
import com.amazonaws.services.dynamodbv2.model.StreamStatus;
import com.amazonaws.services.dynamodbv2.model.StreamViewType;
import com.amazonaws.services.kinesis.model.InvalidArgumentException;
import com.amazonaws.services.kinesis.model.ListShardsRequest;
import com.amazonaws.services.kinesis.model.ListShardsResult;
import com.amazonaws.services.kinesis.model.ResourceNotFoundException;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
/**
* @author Asiel Caballero
*
* @since 2.0.3
*/
public class SpringDynamoDBAdapterClientTests {
private final AmazonDynamoDBStreams amazonDynamoDBStreams = new InMemoryAmazonDynamoDBStreams();
private final SpringDynamoDBAdapterClient springDynamoDBAdapterClient =
new SpringDynamoDBAdapterClient(amazonDynamoDBStreams);
@Test
public void listShardsForNotFoundStream() {
assertThatExceptionOfType(ResourceNotFoundException.class)
.isThrownBy(() -> springDynamoDBAdapterClient.listShards(new ListShardsRequest()
.withStreamName("not-found")));
}
@Test
public void listShardsWithInvalidToken() {
assertThatExceptionOfType(InvalidArgumentException.class)
.isThrownBy(() -> springDynamoDBAdapterClient.listShards(new ListShardsRequest()
.withNextToken("invalid-token")));
}
@Test
public void listShardsWithTokenAndStreamName() {
assertThatExceptionOfType(InvalidArgumentException.class)
.isThrownBy(() ->
springDynamoDBAdapterClient.listShards(new ListShardsRequest()
.withStreamName(InMemoryAmazonDynamoDBStreams.STREAM_ARN)
.withNextToken("valid!!##%%token")));
}
@Test
public void listShardsNoPagination() {
ListShardsResult listShards = springDynamoDBAdapterClient.listShards(new ListShardsRequest()
.withStreamName(InMemoryAmazonDynamoDBStreams.STREAM_ARN));
assertThat(listShards.getNextToken()).isNull();
assertThat(listShards.getShards().size()).isEqualTo(InMemoryAmazonDynamoDBStreams.SHARDS.size());
}
@Test
public void listShardWithTokenPagination() {
int maxResults = (InMemoryAmazonDynamoDBStreams.SHARDS.size() / 2)
+ (InMemoryAmazonDynamoDBStreams.SHARDS.size() % 2);
ListShardsResult listShards = springDynamoDBAdapterClient.listShards(new ListShardsRequest()
.withStreamName(InMemoryAmazonDynamoDBStreams.STREAM_ARN)
.withMaxResults(maxResults));
assertThat(listShards.getNextToken()).isNotNull();
assertThat(listShards.getShards().size()).isEqualTo(maxResults);
listShards = springDynamoDBAdapterClient.listShards(new ListShardsRequest()
.withNextToken(listShards.getNextToken()));
assertThat(listShards.getNextToken()).isNull();
assertThat(listShards.getShards().size()).isEqualTo(InMemoryAmazonDynamoDBStreams.SHARDS.size() - maxResults);
}
@Test
public void listShardsWithShardIdPagination() {
ListShardsResult listShards = springDynamoDBAdapterClient.listShards(new ListShardsRequest()
.withStreamName(InMemoryAmazonDynamoDBStreams.STREAM_ARN)
.withExclusiveStartShardId(InMemoryAmazonDynamoDBStreams.SHARDS.get(2).getShardId())
.withMaxResults(1));
assertThat(listShards.getNextToken()).isNotNull();
assertThat(listShards.getShards().size()).isEqualTo(1);
assertThat(listShards.getShards().get(0).getShardId())
.isEqualTo(InMemoryAmazonDynamoDBStreams.SHARDS.get(3).getShardId());
}
private static class InMemoryAmazonDynamoDBStreams extends AbstractAmazonDynamoDBStreams {
private static final String TABLE_NAME = "test-streams";
private static final String STREAM_LABEL = "2020-10-21T11:49:13.355";
private static final String STREAM_ARN = String
.format("arn:aws:dynamodb:%s:%s:table/%s/stream/%s",
Regions.DEFAULT_REGION.getName(), "000000000000", TABLE_NAME, STREAM_LABEL);
private static final List<Shard> SHARDS =
Arrays.asList(
buildShard("shardId-00000001603195033866-c5d0c2b1", "51100000000002515059163", "51300000000002521847055"),
buildShard("shardId-00000001603208699318-b15c42af", "804300000000026046960744", "804300000000026046960744")
.withParentShardId("shardId-00000001603195033866-c5d0c2b1"),
buildShard("shardId-00000001603223404428-90b80e6c", "1613900000000033324335703", "1613900000000033324335703")
.withParentShardId("shardId-00000001603208699318-b15c42af"),
buildShard("shardId-00000001603237029376-bd9c40dd", "2364600000000001701561758", "2364600000000001701561758")
.withParentShardId("shardId-00000001603223404428-90b80e6c"),
buildShard("shardId-00000001603249855034-b917a47f", "3071400000000035046301998", "3071400000000035046301998")
.withParentShardId("shardId-00000001603237029376-bd9c40dd"));
private final StreamDescription streamDescription = new StreamDescription()
.withStreamArn(STREAM_ARN)
.withStreamLabel(STREAM_LABEL)
.withStreamViewType(StreamViewType.KEYS_ONLY)
.withCreationRequestDateTime(new Date())
.withTableName(TABLE_NAME)
.withKeySchema(new KeySchemaElement("name", KeyType.HASH))
.withStreamStatus(StreamStatus.ENABLED);
private static Shard buildShard(String parentShardIdString, String startingSequenceNumber,
String endingSequenceNumber) {
return new Shard()
.withShardId(parentShardIdString)
.withSequenceNumberRange(new SequenceNumberRange()
.withStartingSequenceNumber(startingSequenceNumber)
.withEndingSequenceNumber(endingSequenceNumber));
}
@Override
public DescribeStreamResult describeStream(DescribeStreamRequest request) {
// Invalid StreamArn (Service: AmazonDynamoDBStreams; Status Code: 400;
// Error Code: ValidationException; Request ID: <Request ID>; Proxy: null)
if (!STREAM_ARN.equals(request.getStreamArn())) {
throw new AmazonDynamoDBException("Invalid StreamArn");
}
int limit = request.getLimit() != null ? request.getLimit() : 100;
int shardIndex = request.getExclusiveStartShardId() != null
? findShardIndex(request.getExclusiveStartShardId())
: 0;
int lastShardIndex = Math.min(shardIndex + limit, SHARDS.size());
streamDescription.setShards(SHARDS.subList(shardIndex, lastShardIndex));
streamDescription.setLastEvaluatedShardId(
lastShardIndex != SHARDS.size() ? SHARDS.get(lastShardIndex).getShardId() : null);
return new DescribeStreamResult()
.withStreamDescription(streamDescription);
}
private int findShardIndex(String shardId) {
int i = 0;
// checkstyle forced this way of writing it
while (i < SHARDS.size() && !SHARDS.get(i).getShardId().equals(shardId)) {
i++;
}
if (i + 1 >= SHARDS.size()) {
throw new RuntimeException("ShardId not found");
}
return i + 1;
}
}
}

314
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/test/java/org/springframework/cloud/stream/binder/kinesis/provisioning/KinesisStreamProvisionerTests.java Normal file
View File

@@ -0,0 +1,314 @@
/*
* Copyright 2017-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.binder.kinesis.provisioning;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import com.amazonaws.services.kinesis.AmazonKinesis;
import com.amazonaws.services.kinesis.model.CreateStreamResult;
import com.amazonaws.services.kinesis.model.DescribeStreamRequest;
import com.amazonaws.services.kinesis.model.DescribeStreamResult;
import com.amazonaws.services.kinesis.model.ListShardsRequest;
import com.amazonaws.services.kinesis.model.ListShardsResult;
import com.amazonaws.services.kinesis.model.ResourceNotFoundException;
import com.amazonaws.services.kinesis.model.ScalingType;
import com.amazonaws.services.kinesis.model.Shard;
import com.amazonaws.services.kinesis.model.StreamDescription;
import com.amazonaws.services.kinesis.model.StreamStatus;
import com.amazonaws.services.kinesis.model.UpdateShardCountRequest;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisBinderConfigurationProperties;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisConsumerProperties;
import org.springframework.cloud.stream.binder.kinesis.properties.KinesisProducerProperties;
import org.springframework.cloud.stream.provisioning.ConsumerDestination;
import org.springframework.cloud.stream.provisioning.ProducerDestination;
import org.springframework.cloud.stream.provisioning.ProvisioningException;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Tests for the {@link KinesisStreamProvisioner}.
*
* @author Jacob Severson
* @author Artem Bilan
* @author Sergiu Pantiru
*/
class KinesisStreamProvisionerTests {
@Test
void testProvisionProducerSuccessfulWithExistingStream() {
AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
amazonKinesisMock, binderProperties);
ExtendedProducerProperties<KinesisProducerProperties> extendedProducerProperties =
new ExtendedProducerProperties<>(new KinesisProducerProperties());
String name = "test-stream";
when(amazonKinesisMock.listShards(any(ListShardsRequest.class)))
.thenReturn(new ListShardsResult().withShards(new Shard()));
ProducerDestination destination = provisioner.provisionProducerDestination(name,
extendedProducerProperties);
verify(amazonKinesisMock).listShards(any(ListShardsRequest.class));
assertThat(destination.getName()).isEqualTo(name);
}
@Test
void testProvisionConsumerSuccessfulWithExistingStream() {
AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
amazonKinesisMock, binderProperties);
ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
new ExtendedConsumerProperties<>(
new KinesisConsumerProperties());
String name = "test-stream";
String group = "test-group";
when(amazonKinesisMock.listShards(any(ListShardsRequest.class)))
.thenReturn(new ListShardsResult().withShards(new Shard()));
ConsumerDestination destination = provisioner.provisionConsumerDestination(name,
group, extendedConsumerProperties);
verify(amazonKinesisMock).listShards(any(ListShardsRequest.class));
assertThat(destination.getName()).isEqualTo(name);
}
@Test
void testProvisionConsumerExistingStreamUpdateShards() {
AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
ArgumentCaptor<UpdateShardCountRequest> updateShardCaptor = ArgumentCaptor
.forClass(UpdateShardCountRequest.class);
String name = "test-stream";
String group = "test-group";
int targetShardCount = 2;
KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
binderProperties.setMinShardCount(targetShardCount);
binderProperties.setAutoAddShards(true);
KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
amazonKinesisMock, binderProperties);
ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
new ExtendedConsumerProperties<>(
new KinesisConsumerProperties());
DescribeStreamResult describeOriginalStream = describeStreamResultWithShards(
Collections.singletonList(new Shard()));
DescribeStreamResult describeUpdatedStream = describeStreamResultWithShards(
Arrays.asList(new Shard(), new Shard()));
when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
.thenReturn(describeOriginalStream).thenReturn(describeUpdatedStream);
when(amazonKinesisMock.listShards(any(ListShardsRequest.class)))
.thenReturn(new ListShardsResult().withShards(new Shard()))
.thenReturn(new ListShardsResult().withShards(new Shard(), new Shard()));
provisioner.provisionConsumerDestination(name, group, extendedConsumerProperties);
verify(amazonKinesisMock, times(1)).updateShardCount(updateShardCaptor.capture());
assertThat(updateShardCaptor.getValue().getStreamName()).isEqualTo(name);
assertThat(updateShardCaptor.getValue().getScalingType())
.isEqualTo(ScalingType.UNIFORM_SCALING.name());
assertThat(updateShardCaptor.getValue().getTargetShardCount())
.isEqualTo(targetShardCount);
}
@Test
void testProvisionProducerSuccessfulWithNewStream() {
AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
amazonKinesisMock, binderProperties);
ExtendedProducerProperties<KinesisProducerProperties> extendedProducerProperties =
new ExtendedProducerProperties<>(
new KinesisProducerProperties());
String name = "test-stream";
Integer shards = 1;
when(amazonKinesisMock.listShards(any(ListShardsRequest.class)))
.thenThrow(new ResourceNotFoundException("I got nothing"))
.thenReturn(new ListShardsResult().withShards(new Shard()));
when(amazonKinesisMock.createStream(name, shards))
.thenReturn(new CreateStreamResult());
when(amazonKinesisMock.describeStream(name))
.thenReturn(new DescribeStreamResult()
.withStreamDescription(new StreamDescription()
.withStreamStatus(StreamStatus.ACTIVE)));
ProducerDestination destination = provisioner.provisionProducerDestination(name,
extendedProducerProperties);
verify(amazonKinesisMock)
.listShards(any(ListShardsRequest.class));
verify(amazonKinesisMock).createStream(name, shards);
assertThat(destination.getName()).isEqualTo(name);
}
@Test
void testProvisionProducerUpdateShards() {
AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
ArgumentCaptor<UpdateShardCountRequest> updateShardCaptor = ArgumentCaptor
.forClass(UpdateShardCountRequest.class);
String name = "test-stream";
String group = "test-group";
int targetShardCount = 2;
KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
binderProperties.setMinShardCount(targetShardCount);
binderProperties.setAutoAddShards(true);
KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
amazonKinesisMock, binderProperties);
ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
new ExtendedConsumerProperties<>(
new KinesisConsumerProperties());
DescribeStreamResult describeOriginalStream = describeStreamResultWithShards(
Collections.singletonList(new Shard()));
DescribeStreamResult describeUpdatedStream = describeStreamResultWithShards(
Arrays.asList(new Shard(), new Shard()));
when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
.thenReturn(describeOriginalStream).thenReturn(describeUpdatedStream);
when(amazonKinesisMock.listShards(any(ListShardsRequest.class)))
.thenReturn(new ListShardsResult().withShards(new Shard()))
.thenReturn(new ListShardsResult().withShards(new Shard(), new Shard()));
provisioner.provisionConsumerDestination(name, group, extendedConsumerProperties);
verify(amazonKinesisMock, times(1)).updateShardCount(updateShardCaptor.capture());
assertThat(updateShardCaptor.getValue().getStreamName()).isEqualTo(name);
assertThat(updateShardCaptor.getValue().getScalingType())
.isEqualTo(ScalingType.UNIFORM_SCALING.name());
assertThat(updateShardCaptor.getValue().getTargetShardCount())
.isEqualTo(targetShardCount);
}
@Test
void testProvisionConsumerSuccessfulWithNewStream() {
AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
amazonKinesisMock, binderProperties);
int instanceCount = 1;
int concurrency = 1;
ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
new ExtendedConsumerProperties<>(
new KinesisConsumerProperties());
extendedConsumerProperties.setInstanceCount(instanceCount);
extendedConsumerProperties.setConcurrency(concurrency);
String name = "test-stream";
String group = "test-group";
when(amazonKinesisMock.listShards(any(ListShardsRequest.class)))
.thenThrow(new ResourceNotFoundException("I got nothing"))
.thenReturn(new ListShardsResult().withShards(new Shard()));
when(amazonKinesisMock.createStream(name, instanceCount * concurrency))
.thenReturn(new CreateStreamResult());
when(amazonKinesisMock.describeStream(name))
.thenReturn(new DescribeStreamResult()
.withStreamDescription(new StreamDescription()
.withStreamStatus(StreamStatus.ACTIVE)));
ConsumerDestination destination = provisioner.provisionConsumerDestination(name,
group, extendedConsumerProperties);
verify(amazonKinesisMock, times(1))
.listShards(any(ListShardsRequest.class));
verify(amazonKinesisMock).createStream(name, instanceCount * concurrency);
assertThat(destination.getName()).isEqualTo(name);
}
private static DescribeStreamResult describeStreamResultWithShards(
List<Shard> shards) {
return new DescribeStreamResult().withStreamDescription(new StreamDescription()
.withShards(shards).withStreamStatus(StreamStatus.ACTIVE)
.withHasMoreShards(Boolean.FALSE));
}
@Test
void testProvisionConsumerResourceNotFoundException() {
AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
binderProperties.setAutoCreateStream(false);
KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
amazonKinesisMock, binderProperties);
int instanceCount = 1;
int concurrency = 1;
ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
new ExtendedConsumerProperties<>(
new KinesisConsumerProperties());
extendedConsumerProperties.setInstanceCount(instanceCount);
extendedConsumerProperties.setConcurrency(concurrency);
String name = "test-stream";
String group = "test-group";
when(amazonKinesisMock.listShards(any(ListShardsRequest.class)))
.thenThrow(new ResourceNotFoundException("Stream not found"));
assertThatThrownBy(() -> provisioner.provisionConsumerDestination(name, group,
extendedConsumerProperties))
.isInstanceOf(ProvisioningException.class)
.hasMessageContaining(
"The stream [test-stream] was not found and auto creation is disabled.")
.hasCauseInstanceOf(ResourceNotFoundException.class);
verify(amazonKinesisMock, times(1))
.listShards(any(ListShardsRequest.class));
verify(amazonKinesisMock, never()).createStream(name,
instanceCount * concurrency);
}
}

21
binders/kinesis-binder/spring-cloud-stream-binder-kinesis/src/test/resources/logback-test.xml Normal file
View File

@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<statusListener class="ch.qos.logback.core.status.NopStatusListener" />
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<layout class="ch.qos.logback.classic.PatternLayout">
<Pattern>
%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n
</Pattern>
</layout>
</appender>
<logger name="org.springframework.cloud.stream.binder.kinesis" level="debug"/>
<logger name="org.springframework.integration" level="warn"/>
<root level="warn">
<appender-ref ref="STDOUT" />
</root>
</configuration>

1
binders/pom.xml
View File

@@ -16,6 +16,7 @@
<modules>
<module>kafka-binder</module>
<module>rabbit-binder</module>
<module>kinesis-binder</module>
</modules>
</project>

3
bom/pom.xml
View File

@@ -88,6 +88,9 @@
<profiles>
<profile>
<id>spring</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<repositories>
<repository>
<id>spring-snapshots</id>