diff --git a/.gitignore b/.gitignore index 6143e53..5ceefde 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,27 @@ -# Compiled class file -*.class - -# Log file -*.log - -# BlueJ files -*.ctxt - -# Mobile Tools for Java (J2ME) -.mtj.tmp/ - -# Package Files # -*.jar -*.war -*.ear -*.zip -*.tar.gz -*.rar - -# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml -hs_err_pid* +apps/ +/application.yml +/application.properties +asciidoctor.css +*~ +.#* +*# +target/ +build/ +bin/ +_site/ +.classpath +.project +.settings +.springBeans +.DS_Store +*.sw* +*.iml +*.ipr +*.iws +.idea/ +.factorypath +spring-xd-samples/*/xd +dump.rdb +coverage-error.log +.apt_generated +aws.credentials.properties \ No newline at end of file diff --git a/.mvn/jvm.config b/.mvn/jvm.config new file mode 100644 index 0000000..0e7dabe --- /dev/null +++ b/.mvn/jvm.config @@ -0,0 +1 @@ +-Xmx1024m -XX:CICompilerCount=1 -XX:TieredStopAtLevel=1 -Djava.security.egd=file:/dev/./urandom \ No newline at end of file diff --git a/.mvn/maven.config b/.mvn/maven.config new file mode 100644 index 0000000..3b8cf46 --- /dev/null +++ b/.mvn/maven.config @@ -0,0 +1 @@ +-DaltSnapshotDeploymentRepository=repo.spring.io::default::https://repo.spring.io/libs-snapshot-local -P spring diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar new file mode 100755 index 0000000..9cc84ea Binary files /dev/null and b/.mvn/wrapper/maven-wrapper.jar differ diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100755 index 0000000..56bb016 --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1 @@ +distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.0/apache-maven-3.5.0-bin.zip \ No newline at end of file diff --git a/README.md b/README.md index e1f4b17..62c8a13 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ -# Spring Cloud Stream binder for AWS Kinesis +# Spring Cloud Stream Binder for AWS Kinesis -This repository will contain the code for the Spring Cloud Stream binder implementation for AWS Kinesis. +This repository contains the code for the Spring Cloud Stream Binder implementation for AWS Kinesis. diff --git a/mvnw b/mvnw new file mode 100755 index 0000000..5bf251c --- /dev/null +++ b/mvnw @@ -0,0 +1,225 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Migwn, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +echo $MAVEN_PROJECTBASEDIR +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100755 index 0000000..48c810e --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,143 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" + +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..5f15211 --- /dev/null +++ b/pom.xml @@ -0,0 +1,139 @@ + + + 4.0.0 + + + org.springframework.cloud + spring-cloud-build + 1.3.3.BUILD-SNAPSHOT + + + + spring-cloud-stream-binder-kinesis-parent + 1.0.0.BUILD-SNAPSHOT + pom + + + 1.3.0.BUILD-SNAPSHOT + 1.2.2.BUILD-SNAPSHOT + 1.1.0.BUILD-SNAPSHOT + + + + spring-cloud-stream-binder-kinesis-core + spring-cloud-stream-binder-kinesis + spring-cloud-starter-stream-kinesis + + + + + + org.springframework.cloud + spring-cloud-stream + ${spring-cloud-stream.version} + + + org.springframework.integration + spring-integration-aws + ${spring-integration-aws-version} + + + org.springframework.cloud + spring-cloud-stream-codec + ${spring-cloud-stream.version} + + + org.springframework.cloud + spring-cloud-stream-binder-kinesis + ${project.version} + + + org.springframework.cloud + spring-cloud-stream-binder-kinesis-core + ${project.version} + + + org.springframework.cloud + spring-cloud-stream-binder-test + ${spring-cloud-stream.version} + test + + + org.springframework.cloud + spring-cloud-aws-dependencies + ${spring-cloud-aws.version} + pom + import + + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.7 + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + com.puppycrawl.tools + checkstyle + 7.1 + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + true + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + org.springframework.cloud + spring-cloud-build-tools + 1.3.1.RELEASE + + + + + checkstyle-validation + validate + + checkstyle.xml + UTF-8 + true + false + true + + + check + + + + + + + + diff --git a/spring-cloud-starter-stream-kinesis/pom.xml b/spring-cloud-starter-stream-kinesis/pom.xml new file mode 100644 index 0000000..50d0927 --- /dev/null +++ b/spring-cloud-starter-stream-kinesis/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + + org.springframework.cloud + spring-cloud-stream-binder-kinesis-parent + 1.0.0.BUILD-SNAPSHOT + + + spring-cloud-starter-stream-kinesis + Spring Cloud Starter Stream Kinesis + + + ${basedir}/../.. + + + + + org.springframework.cloud + spring-cloud-stream-binder-kinesis + + + + \ No newline at end of file diff --git a/spring-cloud-stream-binder-kinesis-core/pom.xml b/spring-cloud-stream-binder-kinesis-core/pom.xml new file mode 100644 index 0000000..033917b --- /dev/null +++ b/spring-cloud-stream-binder-kinesis-core/pom.xml @@ -0,0 +1,31 @@ + + + 4.0.0 + + + org.springframework.cloud + spring-cloud-stream-binder-kinesis-parent + 1.0.0.BUILD-SNAPSHOT + + + spring-cloud-stream-binder-kinesis-core + Spring Cloud Stream Kinesis Binder Core + + + + org.springframework.cloud + spring-cloud-stream + + + com.amazonaws + aws-java-sdk-kinesis + + + org.springframework.boot + spring-boot-configuration-processor + true + + + + diff --git a/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisBinderConfigurationProperties.java b/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisBinderConfigurationProperties.java new file mode 100644 index 0000000..2b620ee --- /dev/null +++ b/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisBinderConfigurationProperties.java @@ -0,0 +1,29 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kinesis.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * + * @author Peter Oates + * + */ +@ConfigurationProperties(prefix = "spring.cloud.stream.kinesis.binder") +public class KinesisBinderConfigurationProperties { + +} diff --git a/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisBindingProperties.java b/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisBindingProperties.java new file mode 100644 index 0000000..ef77088 --- /dev/null +++ b/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisBindingProperties.java @@ -0,0 +1,43 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kinesis.properties; + +/** + * @author Peter Oates + */ +public class KinesisBindingProperties { + + private KinesisConsumerProperties consumer = new KinesisConsumerProperties(); + + private KinesisProducerProperties producer = new KinesisProducerProperties(); + + public KinesisConsumerProperties getConsumer() { + return consumer; + } + + public void setConsumer(KinesisConsumerProperties consumer) { + this.consumer = consumer; + } + + public KinesisProducerProperties getProducer() { + return producer; + } + + public void setProducer(KinesisProducerProperties producer) { + this.producer = producer; + } +} diff --git a/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisConsumerProperties.java b/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisConsumerProperties.java new file mode 100644 index 0000000..8cf4749 --- /dev/null +++ b/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisConsumerProperties.java @@ -0,0 +1,26 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kinesis.properties; + +/** + * + * @author Peter Oates + * + */ +public class KinesisConsumerProperties { + +} diff --git a/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisExtendedBindingProperties.java b/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisExtendedBindingProperties.java new file mode 100644 index 0000000..07a225d --- /dev/null +++ b/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisExtendedBindingProperties.java @@ -0,0 +1,64 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kinesis.properties; + +import java.util.HashMap; +import java.util.Map; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.cloud.stream.binder.ExtendedBindingProperties; + +/** + * + * @author Peter Oates + * + */ +@ConfigurationProperties("spring.cloud.stream.kinesis") +public class KinesisExtendedBindingProperties + implements ExtendedBindingProperties { + + private Map bindings = new HashMap<>(); + + public Map getBindings() { + return this.bindings; + } + + public void setBindings(Map bindings) { + this.bindings = bindings; + } + + @Override + public KinesisConsumerProperties getExtendedConsumerProperties(String channelName) { + if (this.bindings.containsKey(channelName) && this.bindings.get(channelName).getConsumer() != null) { + return this.bindings.get(channelName).getConsumer(); + } + else { + return new KinesisConsumerProperties(); + } + } + + @Override + public KinesisProducerProperties getExtendedProducerProperties(String channelName) { + if (this.bindings.containsKey(channelName) && this.bindings.get(channelName).getProducer() != null) { + return this.bindings.get(channelName).getProducer(); + } + else { + return new KinesisProducerProperties(); + } + } + +} diff --git a/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisProducerProperties.java b/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisProducerProperties.java new file mode 100644 index 0000000..39e8896 --- /dev/null +++ b/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/properties/KinesisProducerProperties.java @@ -0,0 +1,26 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kinesis.properties; + +/** + * + * @author Peter Oates + * + */ +public class KinesisProducerProperties { + +} diff --git a/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/provisioning/KinesisStreamProvisioner.java b/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/provisioning/KinesisStreamProvisioner.java new file mode 100644 index 0000000..cdbdaf1 --- /dev/null +++ b/spring-cloud-stream-binder-kinesis-core/src/main/java/org/springframework/cloud/stream/binder/kinesis/provisioning/KinesisStreamProvisioner.java @@ -0,0 +1,154 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kinesis.provisioning; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.kinesis.properties.KinesisBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kinesis.properties.KinesisConsumerProperties; +import org.springframework.cloud.stream.binder.kinesis.properties.KinesisProducerProperties; +import org.springframework.cloud.stream.provisioning.ConsumerDestination; +import org.springframework.cloud.stream.provisioning.ProducerDestination; +import org.springframework.cloud.stream.provisioning.ProvisioningException; +import org.springframework.cloud.stream.provisioning.ProvisioningProvider; +import org.springframework.util.Assert; + +import com.amazonaws.services.kinesis.AmazonKinesis; + +/** + * + * @author Peter Oates + * @author Artem Bilan + * + */ +public class KinesisStreamProvisioner + implements ProvisioningProvider, + ExtendedProducerProperties>, InitializingBean { + + private final AmazonKinesis amazonKinesis; + + private final KinesisBinderConfigurationProperties configurationProperties; + + public KinesisStreamProvisioner(AmazonKinesis amazonKinesis, + KinesisBinderConfigurationProperties kinesisBinderConfigurationProperties) { + Assert.notNull(amazonKinesis, "'amazonKinesis' must not be null"); + Assert.notNull(kinesisBinderConfigurationProperties, "'kinesisBinderConfigurationProperties' must not be null"); + this.amazonKinesis = amazonKinesis; + this.configurationProperties = kinesisBinderConfigurationProperties; + } + + @Override + public void afterPropertiesSet() throws Exception { + // TODO Auto-generated method stub + + } + + @Override + public ProducerDestination provisionProducerDestination(String name, + ExtendedProducerProperties properties) throws ProvisioningException { + + KinesisProducerDestination producer = new KinesisProducerDestination(name); + + return producer; + } + + @Override + public ConsumerDestination provisionConsumerDestination(String name, String group, + ExtendedConsumerProperties properties) throws ProvisioningException { + + KinesisConsumerDestination consumer = new KinesisConsumerDestination(name); + + return consumer; + } + + + private static final class KinesisProducerDestination implements ProducerDestination { + + private final String streamName; + + private final int shards; + + KinesisProducerDestination(String streamName) { + this(streamName, 0); + } + + KinesisProducerDestination(String streamName, Integer shards) { + this.streamName = streamName; + this.shards = shards; + } + + @Override + public String getName() { + return this.streamName; + } + + @Override + public String getNameForPartition(int shard) { + return this.streamName; + } + + @Override + public String toString() { + return "KinesisProducerDestination{" + + "streamName='" + this.streamName + '\'' + + ", shards=" + this.shards + + '}'; + } + + } + + + private static final class KinesisConsumerDestination implements ConsumerDestination { + + private final String streamName; + + private final int shards; + + private final String dlqName; + + KinesisConsumerDestination(String streamName) { + this(streamName, 0, null); + } + + KinesisConsumerDestination(String streamName, int shards) { + this(streamName, shards, null); + } + + KinesisConsumerDestination(String streamName, Integer shards, String dlqName) { + this.streamName = streamName; + this.shards = shards; + this.dlqName = dlqName; + } + + @Override + public String getName() { + return this.streamName; + } + + @Override + public String toString() { + return "KinesisConsumerDestination{" + + "streamName='" + streamName + '\'' + + ", shards=" + shards + + ", dlqName='" + dlqName + '\'' + + '}'; + } + + } + +} diff --git a/spring-cloud-stream-binder-kinesis/pom.xml b/spring-cloud-stream-binder-kinesis/pom.xml new file mode 100644 index 0000000..d671ccd --- /dev/null +++ b/spring-cloud-stream-binder-kinesis/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + + + org.springframework.cloud + spring-cloud-stream-binder-kinesis-parent + 1.0.0.BUILD-SNAPSHOT + + + spring-cloud-stream-binder-kinesis + jar + spring-cloud-stream-binder-kinesis + AWS Kinesis Binder Implementation + + + + + org.springframework.cloud + spring-cloud-stream-binder-kinesis-core + + + org.springframework.cloud + spring-cloud-stream-codec + + + org.springframework.integration + spring-integration-aws + + + org.springframework.cloud + spring-cloud-starter-aws + + + + diff --git a/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/KinesisMessageChannelBinder.java b/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/KinesisMessageChannelBinder.java new file mode 100644 index 0000000..2b1f35c --- /dev/null +++ b/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/KinesisMessageChannelBinder.java @@ -0,0 +1,118 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kinesis; + +import org.springframework.cloud.stream.binder.AbstractMessageChannelBinder; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder; +import org.springframework.cloud.stream.binder.kinesis.properties.KinesisBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kinesis.properties.KinesisConsumerProperties; +import org.springframework.cloud.stream.binder.kinesis.properties.KinesisExtendedBindingProperties; +import org.springframework.cloud.stream.binder.kinesis.properties.KinesisProducerProperties; +import org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner; +import org.springframework.cloud.stream.provisioning.ConsumerDestination; +import org.springframework.cloud.stream.provisioning.ProducerDestination; +import org.springframework.integration.aws.inbound.kinesis.CheckpointMode; +import org.springframework.integration.aws.inbound.kinesis.KinesisMessageDrivenChannelAdapter; +import org.springframework.integration.aws.inbound.kinesis.KinesisShardOffset; +import org.springframework.integration.aws.inbound.kinesis.ListenerMode; +import org.springframework.integration.aws.outbound.KinesisMessageHandler; +import org.springframework.integration.core.MessageProducer; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; +import org.springframework.util.Assert; + +import com.amazonaws.services.kinesis.AmazonKinesisAsync; + +/** + * + * @author Peter Oates + * @author Artem Bilan + */ +public class KinesisMessageChannelBinder extends + AbstractMessageChannelBinder, ExtendedProducerProperties, KinesisStreamProvisioner> + implements ExtendedPropertiesBinder { + + // not using currently but expect to include global properties here if + // required + private final KinesisBinderConfigurationProperties configurationProperties; + + // not currently utilising any extended properties - not sure if I will need + // these + private KinesisExtendedBindingProperties extendedBindingProperties = new KinesisExtendedBindingProperties(); + + private final AmazonKinesisAsync amazonKinesis; + + public KinesisMessageChannelBinder(AmazonKinesisAsync amazonKinesis, + KinesisBinderConfigurationProperties configurationProperties, + KinesisStreamProvisioner provisioningProvider) { + super(false, null, provisioningProvider); + Assert.notNull(amazonKinesis, "'amazonKinesis' must not be null"); + Assert.notNull(configurationProperties, "'configurationProperties' must not be null"); + this.configurationProperties = configurationProperties; + this.amazonKinesis = amazonKinesis; + } + + @Override + public KinesisConsumerProperties getExtendedConsumerProperties(String channelName) { + return this.extendedBindingProperties.getExtendedConsumerProperties(channelName); + } + + @Override + public KinesisProducerProperties getExtendedProducerProperties(String channelName) { + return this.extendedBindingProperties.getExtendedProducerProperties(channelName); + } + + // below are the main methods to implement - these will create the message + // handlers used by the application + // to put and consume messages + @Override + protected MessageHandler createProducerMessageHandler(ProducerDestination destination, + ExtendedProducerProperties producerProperties) throws Exception { + + KinesisMessageHandler kinesisMessageHandler = new KinesisMessageHandler(this.amazonKinesis); + + kinesisMessageHandler.setStream(destination.getName()); + // tidy up partition key + kinesisMessageHandler.setPartitionKey("name"); + kinesisMessageHandler.setBeanFactory(this.getBeanFactory()); + + return kinesisMessageHandler; + } + + @Override + protected MessageProducer createConsumerEndpoint(ConsumerDestination destination, String group, + ExtendedConsumerProperties properties) throws Exception { + + KinesisMessageDrivenChannelAdapter adapter = + new KinesisMessageDrivenChannelAdapter(this.amazonKinesis, destination.getName()); + + // explicitly setting the offset - LATEST is the default but added here + // so can be configured later + adapter.setStreamInitialSequence(KinesisShardOffset.latest()); + // need to move these properties to the appropriate properties class + adapter.setCheckpointMode(CheckpointMode.record); + adapter.setListenerMode(ListenerMode.record); + adapter.setStartTimeout(10000); + adapter.setDescribeStreamRetries(1); + adapter.setConcurrency(10); + + return adapter; + } + +} diff --git a/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/config/KinesisBinderConfiguration.java b/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/config/KinesisBinderConfiguration.java new file mode 100644 index 0000000..21c1eaf --- /dev/null +++ b/spring-cloud-stream-binder-kinesis/src/main/java/org/springframework/cloud/stream/binder/kinesis/config/KinesisBinderConfiguration.java @@ -0,0 +1,80 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kinesis.config; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.aws.core.region.RegionProvider; +import org.springframework.cloud.stream.binder.Binder; +import org.springframework.cloud.stream.binder.kinesis.KinesisMessageChannelBinder; +import org.springframework.cloud.stream.binder.kinesis.properties.KinesisBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.kinesis.properties.KinesisExtendedBindingProperties; +import org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.integration.codec.Codec; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.services.kinesis.AmazonKinesisAsync; +import com.amazonaws.services.kinesis.AmazonKinesisAsyncClientBuilder; + +/** + * + * @author Peter Oates + * @author Artem Bilan + * + */ +@Configuration +@ConditionalOnMissingBean(Binder.class) +@EnableConfigurationProperties({ KinesisBinderConfigurationProperties.class, KinesisExtendedBindingProperties.class }) +public class KinesisBinderConfiguration { + + @Autowired + private KinesisBinderConfigurationProperties configurationProperties; + + @Bean + public AmazonKinesisAsync amazonKinesis( + AWSCredentialsProvider awsCredentialsProvider, + RegionProvider regionProvider) { + + return AmazonKinesisAsyncClientBuilder.standard() + .withCredentials(awsCredentialsProvider) + .withRegion(regionProvider.getRegion().getName()) + .build(); + } + + @Bean + public KinesisStreamProvisioner provisioningProvider(AmazonKinesisAsync amazonKinesis) { + return new KinesisStreamProvisioner(amazonKinesis, this.configurationProperties); + } + + @Bean + KinesisMessageChannelBinder kinesisMessageChannelBinder( + AmazonKinesisAsync amazonKinesis, + KinesisStreamProvisioner provisioningProvider, + Codec codec) { + + KinesisMessageChannelBinder kinesisMessageChannelBinder = + new KinesisMessageChannelBinder(amazonKinesis, this.configurationProperties, provisioningProvider); + kinesisMessageChannelBinder.setCodec(codec); + + return kinesisMessageChannelBinder; + } + + +} diff --git a/spring-cloud-stream-binder-kinesis/src/main/resources/META-INF/spring.binders b/spring-cloud-stream-binder-kinesis/src/main/resources/META-INF/spring.binders new file mode 100644 index 0000000..83d6d1b --- /dev/null +++ b/spring-cloud-stream-binder-kinesis/src/main/resources/META-INF/spring.binders @@ -0,0 +1 @@ +kinesis: org.springframework.cloud.stream.binder.kinesis.config.KinesisBinderConfiguration