diff --git a/samples/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/src/main/java/sample/consumer/ConfluentAvroConsumerApplication.java b/samples/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/src/main/java/sample/consumer/ConfluentAvroConsumerApplication.java index c727a96c9..8dc55bc21 100644 --- a/samples/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/src/main/java/sample/consumer/ConfluentAvroConsumerApplication.java +++ b/samples/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/src/main/java/sample/consumer/ConfluentAvroConsumerApplication.java @@ -1,3 +1,19 @@ +/* + * Copyright 2020-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package sample.consumer; import java.util.function.Consumer; diff --git a/samples/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/src/main/java/sample/producer1/ConfluentAvroProducer1Application.java b/samples/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/src/main/java/sample/producer1/ConfluentAvroProducer1Application.java index 6dc0a6e16..59e45d03b 100644 --- a/samples/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/src/main/java/sample/producer1/ConfluentAvroProducer1Application.java +++ b/samples/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/src/main/java/sample/producer1/ConfluentAvroProducer1Application.java @@ -1,3 +1,19 @@ +/* + * Copyright 2020-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package sample.producer1; import java.util.Random; @@ -40,6 +56,3 @@ public class ConfluentAvroProducer1Application { return sensor; } } - - - diff --git a/samples/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/src/main/java/sample/producer2/ConfluentAvroProducer2Application.java b/samples/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/src/main/java/sample/producer2/ConfluentAvroProducer2Application.java index bbc19465f..1c3bd90c0 100644 --- a/samples/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/src/main/java/sample/producer2/ConfluentAvroProducer2Application.java +++ b/samples/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/src/main/java/sample/producer2/ConfluentAvroProducer2Application.java @@ -1,3 +1,19 @@ +/* + * Copyright 2020-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package sample.producer2; import java.util.Random; @@ -42,4 +58,3 @@ public class ConfluentAvroProducer2Application { return sensor; } } - diff --git a/samples/confluent-schema-registry-integration/pom.xml b/samples/confluent-schema-registry-integration/pom.xml index 936c87b3f..d676915be 100644 --- a/samples/confluent-schema-registry-integration/pom.xml +++ b/samples/confluent-schema-registry-integration/pom.xml @@ -42,20 +42,6 @@ - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.springframework.boot - spring-boot-maven-plugin - - diff --git a/samples/kafka-native-serialization/src/test/java/com/example/kafkanativeserialization/MyJsonDeserializer.java b/samples/kafka-native-serialization/src/test/java/com/example/kafkanativeserialization/MyJsonDeserializer.java index 2f434d1d2..b2532ebf3 100644 --- a/samples/kafka-native-serialization/src/test/java/com/example/kafkanativeserialization/MyJsonDeserializer.java +++ b/samples/kafka-native-serialization/src/test/java/com/example/kafkanativeserialization/MyJsonDeserializer.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 the original author or authors. + * Copyright 2020-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/samples/kafka-streams-branching/README.adoc b/samples/kafka-streams-branching/README.adoc new file mode 100644 index 000000000..a6d4d903d --- /dev/null +++ b/samples/kafka-streams-branching/README.adoc @@ -0,0 +1,110 @@ +== Spring Cloud Stream Kafka Streams with branching + +This sample demonstrates a Spring Cloud Stream processor using Kafka Streams branching support. + +=== Application +The app is based on the word count application from the Confluent https://github.com/confluentinc/kafka-streams-examples/blob/5.3.0-post/src/main/java/io/confluent/examples/streams/WordCountLambdaExample.java[Kafka Streams examples] and does the following: + +In essence, the application receives text messages from an input topic, filters them by language (English, French, Spanish +and ignoring the rest), and computes word occurrence counts in a configurable time window and finally report that in +the corresponding output topics. + +It uses a single input topic and 3 output topics. + +[[build-app]] +=== Building +To build the app simply execute the following command: +[source,bash] +---- +./mvnw clean install +---- + +=== Running +The sample is exercised via the provided link:./src/test/java/com/example/kafka/streams/branching/KafkaStreamsBranchingSampleTests.java[@SpringBootTest]. +However, it can also be run manually as follows: + +==== Ensure these pre-requisites +**** +* The app has been built by following the <> steps +* Apache Kafka broker available at `localhost:9092` + +[[kafka-tools]] +TIP: The included link:../../../tools/kafka/docker-compose/README.adoc#_all_the_things[Kafka tools] can be used to easily start a broker at the required coordinates +**** + +==== Start the streams app +[source,bash] +---- +java -jar target/kafka-streams-branching-4.0.0-SNAPSHOT.jar +---- + +===== Start output topic consumers +Leverage the Kafka command line tool `kafka-console-consumer` to watch events on each output topic. + +Issue the following commands, each on a separate terminal: + +NOTE: The commands reference `broker1` and `broker1:29091` as they assume the aforementioned <> are used to create the cluster. If you start up your own cluster you will need to adjust the coordinates accordingly. + +NOTE: Also note that the port is `29091` as that is the internal port configured on the cluster (rather than the expected `9091`) +[source,bash] +---- +docker exec -it broker1 /bin/kafka-console-consumer --bootstrap-server broker1:29091 --topic english-counts +---- + +[source,bash] +---- +docker exec -it broker1 /bin/kafka-console-consumer --bootstrap-server broker1:29091 --topic french-counts +---- + +[source,bash] +---- +docker exec -it broker1 /bin/kafka-console-consumer --bootstrap-server broker1:29091 --topic spanish-counts +---- +At this point each of the consumers is just waiting for events to be sent to the output topics. + +===== Send input messages +Leverage the Kafka command line tool `kafka-console-producer` to send messges (words) to the input topic. + +Issue the following command on a separate terminal: + +[source,bash] +---- +docker exec -it broker1 /bin/kafka-console-producer --broker-list broker1:29091 --topic words +---- + +Type one of the expected words and click "". Repeat a few times. Something like the following: + +[source,bash] +---- +>english +>english +>english +>french +>french +>foo +>spanish +---- + +===== View output messages +As you enter words in the console producer (input into Kafka streams processor) the corresponding console consumer will log the processed word count event (output from the Kafka Streams processor). + +The word "english" goes to topic english-counts, "french" goes to topic french-counts and "spanish" goes to spanish-counts. Something like the following: + +.english-counts +[source,bash] +---- +{"word":"english","count":1,"start":1654923396000,"end":1654923402000} +{"word":"english","count":1,"start":1654923402000,"end":1654923408000} +{"word":"english","count":1,"start":1654923422000,"end":1654923424000} +---- +.french-counts +[source,bash] +---- +{"word":"french","count":1,"start":1654923396000,"end":1654923402000} +{"word":"french","count":1,"start":1654923402000,"end":1654923408000} +---- +.spanish-counts +[source,bash] +---- +{"word":"spanish","count":1,"start":1654923396000,"end":1654923402000} +---- diff --git a/samples/kafka-streams-branching/mvnw b/samples/kafka-streams-branching/mvnw new file mode 100755 index 000000000..a16b5431b --- /dev/null +++ b/samples/kafka-streams-branching/mvnw @@ -0,0 +1,310 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + if [ -n "$MVNW_REPOURL" ]; then + jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + else + jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + fi + while IFS="=" read key value; do + case "$key" in (wrapperUrl) jarUrl="$value"; break ;; + esac + done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + if $cygwin; then + wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` + fi + + if command -v wget > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget "$jarUrl" -O "$wrapperJarPath" + else + wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl -o "$wrapperJarPath" "$jarUrl" -f + else + curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f + fi + + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaClass=`cygpath --path --windows "$javaClass"` + fi + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/samples/kafka-streams-branching/mvnw.cmd b/samples/kafka-streams-branching/mvnw.cmd new file mode 100644 index 000000000..c8d43372c --- /dev/null +++ b/samples/kafka-streams-branching/mvnw.cmd @@ -0,0 +1,182 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + +FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/samples/kafka-streams-branching/pom.xml b/samples/kafka-streams-branching/pom.xml new file mode 100644 index 000000000..d9e72acab --- /dev/null +++ b/samples/kafka-streams-branching/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + com.example + kafka-streams-branching + kafka-streams-branching + + + org.springframework.cloud + spring-cloud-stream-samples-parent + 4.0.0-SNAPSHOT + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.cloud + spring-cloud-stream-binder-kafka-streams + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.kafka + spring-kafka-test + test + + + + diff --git a/samples/kafka-streams-branching/src/main/java/com/example/kafka/streams/branching/KafkaStreamsBranchingSample.java b/samples/kafka-streams-branching/src/main/java/com/example/kafka/streams/branching/KafkaStreamsBranchingSample.java new file mode 100644 index 000000000..26928528d --- /dev/null +++ b/samples/kafka-streams-branching/src/main/java/com/example/kafka/streams/branching/KafkaStreamsBranchingSample.java @@ -0,0 +1,117 @@ +/* + * Copyright 2020-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.kafka.streams.branching; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Date; +import java.util.function.Function; + +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.TimeWindows; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; + +@SpringBootApplication(proxyBeanMethods = false) +class KafkaStreamsBranchingSample { + + public static void main(String[] args) { + SpringApplication.run(KafkaStreamsBranchingSample.class, args); + } + + public static class WordCountProcessorApplication { + + @Bean + @SuppressWarnings("unchecked") + public Function, KStream[]> process() { + + Predicate isEnglish = (k, v) -> v.word.equals("english"); + Predicate isFrench = (k, v) -> v.word.equals("french"); + Predicate isSpanish = (k, v) -> v.word.equals("spanish"); + + return input -> input + .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .groupBy((key, value) -> value) + .windowedBy(TimeWindows.of(Duration.ofSeconds(6))) + .count(Materialized.as("WordCounts-1")) + .toStream() + .map((key, value) -> new KeyValue<>(null, + new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))) + .split() + .branch(isEnglish) + .branch(isFrench) + .branch(isSpanish) + .noDefaultBranch() + .values().toArray(new KStream[0]); + } + } + + static class WordCount { + + private String word; + + private long count; + + private Date start; + + private Date end; + + WordCount(String word, long count, Date start, Date end) { + this.word = word; + this.count = count; + this.start = start; + this.end = end; + } + + public String getWord() { + return word; + } + + public void setWord(String word) { + this.word = word; + } + + public long getCount() { + return count; + } + + public void setCount(long count) { + this.count = count; + } + + public Date getStart() { + return start; + } + + public void setStart(Date start) { + this.start = start; + } + + public Date getEnd() { + return end; + } + + public void setEnd(Date end) { + this.end = end; + } + } +} diff --git a/samples/kafka-streams-branching/src/main/resources/application.yml b/samples/kafka-streams-branching/src/main/resources/application.yml new file mode 100644 index 000000000..19b9229ee --- /dev/null +++ b/samples/kafka-streams-branching/src/main/resources/application.yml @@ -0,0 +1,18 @@ +spring: + application.name: kafka-streams-branching-sample + cloud: + stream: + kafka.streams.binder: + configuration: + commit.interval.ms: 1000 + default.key.serde: org.apache.kafka.common.serialization.Serdes$StringSerde + default.value.serde: org.apache.kafka.common.serialization.Serdes$StringSerde + bindings: + process-in-0: + destination: words + process-out-0: + destination: english-counts + process-out-1: + destination: french-counts + process-out-2: + destination: spanish-counts diff --git a/samples/kafka-streams-branching/src/test/java/com/example/kafka/streams/branching/KafkaStreamsBranchingSampleTests.java b/samples/kafka-streams-branching/src/test/java/com/example/kafka/streams/branching/KafkaStreamsBranchingSampleTests.java new file mode 100644 index 000000000..106a7a329 --- /dev/null +++ b/samples/kafka-streams-branching/src/test/java/com/example/kafka/streams/branching/KafkaStreamsBranchingSampleTests.java @@ -0,0 +1,89 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.kafka.streams.branching; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.kafka.config.StreamsBuilderFactoryBean; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.kafka.test.utils.KafkaTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE) +@EmbeddedKafka( + topics = { "words", "english-counts", "french-counts", "spanish-counts" }, + bootstrapServersProperty = "spring.cloud.stream.kafka.streams.binder.brokers", + controlledShutdown = true) +class KafkaStreamsBranchingSampleTests { + + @BeforeEach + void setKafkaStreamsCloseTimeout(@Autowired StreamsBuilderFactoryBean streamsBuilderFactoryBean) { + streamsBuilderFactoryBean.setCloseTimeout(0); + } + + @Test + void wordEventsDirectedToProperTopic(@Autowired EmbeddedKafkaBroker embeddedKafka) throws InterruptedException { + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("words"); + template.sendDefault("english"); + template.sendDefault("french"); + template.sendDefault("spanish"); + } + finally { + pf.destroy(); + } + + TimeUnit.SECONDS.sleep(2); + + Map consumerProps = KafkaTestUtils.consumerProps("wordEventsTest", "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + try (Consumer consumer = cf.createConsumer()) { + embeddedKafka.consumeFromEmbeddedTopics(consumer, "english-counts", "french-counts", "spanish-counts"); + assertThat(KafkaTestUtils.getSingleRecord(consumer, "english-counts", 5000L)) + .extracting(ConsumerRecord::value) + .asInstanceOf(InstanceOfAssertFactories.STRING) + .contains("english"); + assertThat(KafkaTestUtils.getSingleRecord(consumer, "french-counts", 5000L)) + .extracting(ConsumerRecord::value) + .asInstanceOf(InstanceOfAssertFactories.STRING) + .contains("french"); + assertThat(KafkaTestUtils.getSingleRecord(consumer, "spanish-counts", 5000L)) + .extracting(ConsumerRecord::value) + .asInstanceOf(InstanceOfAssertFactories.STRING) + .contains("spanish"); + } + } +} diff --git a/samples/pom.xml b/samples/pom.xml index 98e39b510..4552c1a41 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -24,6 +24,7 @@ confluent-schema-registry-integration kafka-native-serialization + kafka-streams-branching stream-bridge-avro @@ -48,6 +49,10 @@ true + + org.springframework.boot + spring-boot-maven-plugin + diff --git a/samples/stream-bridge-avro/pom.xml b/samples/stream-bridge-avro/pom.xml index 7467d5dd0..5daa676d4 100644 --- a/samples/stream-bridge-avro/pom.xml +++ b/samples/stream-bridge-avro/pom.xml @@ -67,10 +67,6 @@ - - org.springframework.boot - spring-boot-maven-plugin - diff --git a/samples/stream-bridge-avro/src/main/java/com/example/stream/bridge/avro/StreamBridgeAvroApplication.java b/samples/stream-bridge-avro/src/main/java/com/example/stream/bridge/avro/StreamBridgeAvroApplication.java index 121cb5429..532279a88 100644 --- a/samples/stream-bridge-avro/src/main/java/com/example/stream/bridge/avro/StreamBridgeAvroApplication.java +++ b/samples/stream-bridge-avro/src/main/java/com/example/stream/bridge/avro/StreamBridgeAvroApplication.java @@ -1,3 +1,19 @@ +/* + * Copyright 2020-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.example.stream.bridge.avro; import java.util.Random;