diff --git a/kafka-streams/kafka-streams-interactive-query/.gitignore b/kafka-streams/kafka-streams-interactive-query/.gitignore new file mode 100644 index 0000000..2af7cef --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/.gitignore @@ -0,0 +1,24 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +nbproject/private/ +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ \ No newline at end of file diff --git a/kafka-streams/kafka-streams-interactive-query/.mvn/wrapper/maven-wrapper.jar b/kafka-streams/kafka-streams-interactive-query/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000..9cc84ea Binary files /dev/null and b/kafka-streams/kafka-streams-interactive-query/.mvn/wrapper/maven-wrapper.jar differ diff --git a/kafka-streams/kafka-streams-interactive-query/.mvn/wrapper/maven-wrapper.properties b/kafka-streams/kafka-streams-interactive-query/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..c315043 --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1 @@ +distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.0/apache-maven-3.5.0-bin.zip diff --git a/kafka-streams/kafka-streams-interactive-query/README.adoc b/kafka-streams/kafka-streams-interactive-query/README.adoc new file mode 100644 index 0000000..6ecfb4c --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/README.adoc @@ -0,0 +1,38 @@ +== What is this app? + +This is an example of a Spring Cloud Stream processor using Kafka Streams support. + +This example is a Spring Cloud Stream adaptation of this Kafka Streams sample: https://github.com/confluentinc/kafka-streams-examples/tree/4.0.0-post/src/main/java/io/confluent/examples/streams/interactivequeries/kafkamusic + +This sample demonstrates the concept of interactive queries in kafka streams. +There is a REST service provided as part of the application that can be used to query the store interactively. + +==== Starting Kafka in a docker container + +* Skip these steps if you already have a non-Docker Kafka environment. + +1. Go to the docker directory in this repo and invoke the command `docker-compose up -d`. +2. Ensure that in the docker directory and then invoke the script `start-kafka-shell.sh` + +=== Running the app: + +1. Start the confluent schema registry: The following command is based on the confluent platform. + +`./bin/schema-registry-start ./etc/schema-registry/schema-registry.properties` + +2. Go to the root of the repository and do: `./mvnw clean package` + +`java -jar target/kafka-streams-interactive-query-0.0.1-SNAPSHOT.jar` + +3. Run the stand-alone `Producers` application to generate data and start the processing. +Keep it running for a while. + +4. Go to the URL: http://localhost:8080/charts/top-five?genre=Punk +keep refreshing the URL and you will see the song play count information changes. + +* By default we use the docker container IP (mac osx specific) in the `application.yml` for Kafka broker and zookeeper. +Change it in `application.yml` (which requires a rebuild) or pass them as runtime arguments as below. + +`spring.cloud.stream.kstream.binder.brokers=` + +`spring.cloud.stream.kstream.binder.zkNodes=` + diff --git a/kafka-streams/kafka-streams-interactive-query/docker/docker-compose.yml b/kafka-streams/kafka-streams-interactive-query/docker/docker-compose.yml new file mode 100644 index 0000000..ecf41fb --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/docker/docker-compose.yml @@ -0,0 +1,14 @@ +version: '2' +services: + zookeeper: + image: wurstmeister/zookeeper + ports: + - "2181:2181" + kafka: + image: wurstmeister/kafka + ports: + - "9092:9092" + environment: + KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100 + KAFKA_ADVERTISED_PORT: 9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 diff --git a/kafka-streams/kafka-streams-interactive-query/docker/start-kafka-shell.sh b/kafka-streams/kafka-streams-interactive-query/docker/start-kafka-shell.sh new file mode 100755 index 0000000..62663e4 --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/docker/start-kafka-shell.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e HOST_IP=$1 -e ZK=$2 -i -t wurstmeister/kafka /bin/bash diff --git a/kafka-streams/kafka-streams-interactive-query/mvnw b/kafka-streams/kafka-streams-interactive-query/mvnw new file mode 100755 index 0000000..5bf251c --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/mvnw @@ -0,0 +1,225 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Migwn, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +echo $MAVEN_PROJECTBASEDIR +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/kafka-streams/kafka-streams-interactive-query/mvnw.cmd b/kafka-streams/kafka-streams-interactive-query/mvnw.cmd new file mode 100644 index 0000000..019bd74 --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/mvnw.cmd @@ -0,0 +1,143 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" + +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/kafka-streams/kafka-streams-interactive-query/pom.xml b/kafka-streams/kafka-streams-interactive-query/pom.xml new file mode 100644 index 0000000..17077fa --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/pom.xml @@ -0,0 +1,144 @@ + + + 4.0.0 + + kafka.streams.interactive.query + kafka-streams-interactive-query + 0.0.1-SNAPSHOT + jar + + kafka-streams-interactive-query + Demo project for Spring Boot + + + org.springframework.boot + spring-boot-starter-parent + 2.0.0.BUILD-SNAPSHOT + + + + + UTF-8 + UTF-8 + 1.8 + Finchley.BUILD-SNAPSHOT + 4.0.0 + 1.8.2 + + + + + io.confluent + kafka-streams-avro-serde + ${confluent.version} + + + org.slf4j + slf4j-log4j12 + + + + + io.confluent + kafka-avro-serializer + ${confluent.version} + + + io.confluent + kafka-schema-registry-client + ${confluent.version} + + + org.springframework.boot + spring-boot-starter + + + org.springframework.kafka + spring-kafka + 2.1.3.BUILD-SNAPSHOT + + + org.springframework.cloud + spring-cloud-stream-binder-kafka-streams + 2.0.0.BUILD-SNAPSHOT + + + + org.springframework.boot + spring-boot-starter-web + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.springframework.cloud + spring-cloud-dependencies + ${spring-cloud.version} + pom + import + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.avro + avro-maven-plugin + ${avro.version} + + + generate-sources + + schema + + + src/main/resources/avro/kafka/streams/interactive/query + ${project.build.directory}/generated-sources + String + + + + + + + + + + + spring-snapshots + Spring Snapshots + http://repo.spring.io/libs-snapshot-local + + true + + + + spring-milestones + Spring Milestones + http://repo.spring.io/libs-milestone-local + + false + + + + confluent + http://packages.confluent.io/maven/ + + + + + + diff --git a/kafka-streams/kafka-streams-interactive-query/src/main/java/kafka/streams/interactive/query/KafkaStreamsInteractiveQuerySample.java b/kafka-streams/kafka-streams-interactive-query/src/main/java/kafka/streams/interactive/query/KafkaStreamsInteractiveQuerySample.java new file mode 100644 index 0000000..9bf407a --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/src/main/java/kafka/streams/interactive/query/KafkaStreamsInteractiveQuerySample.java @@ -0,0 +1,310 @@ +package kafka.streams.interactive.query; + +import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; +import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; +import kafka.streams.interactive.query.avro.PlayEvent; +import kafka.streams.interactive.query.avro.Song; +import kafka.streams.interactive.query.avro.SongPlayCount; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.*; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.QueryableStoreTypes; +import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.binder.kafka.streams.QueryableStoreRegistry; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.io.*; +import java.util.*; + +@SpringBootApplication +public class KafkaStreamsInteractiveQuerySample { + static final String PLAY_EVENTS = "play-events"; + static final String SONG_FEED = "song-feed"; + static final String TOP_FIVE_KEY = "all"; + static final String TOP_FIVE_SONGS_STORE = "top-five-songs"; + static final String ALL_SONGS = "all-songs"; + + @Autowired + private QueryableStoreRegistry queryableStoreRegistry; + + public static void main(String[] args) { + SpringApplication.run(KafkaStreamsInteractiveQuerySample.class, args); + } + + @EnableBinding(KStreamProcessorX.class) + @EnableAutoConfiguration + public static class KStreamMusicSampleApplication { + + private static final Long MIN_CHARTABLE_DURATION = 30 * 1000L; + private static final String SONG_PLAY_COUNT_STORE = "song-play-count"; + + static final String TOP_FIVE_SONGS_BY_GENRE_STORE = "top-five-songs-by-genre"; + static final String TOP_FIVE_KEY = "all"; + + @StreamListener + public void process(@Input("input") KStream playEvents, + @Input("inputX") KTable songTable) { + + // create and configure the SpecificAvroSerdes required in this example + final Map serdeConfig = Collections.singletonMap( + AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081"); + + final SpecificAvroSerde playEventSerde = new SpecificAvroSerde<>(); + playEventSerde.configure(serdeConfig, false); + + final SpecificAvroSerde keySongSerde = new SpecificAvroSerde<>(); + keySongSerde.configure(serdeConfig, true); + + final SpecificAvroSerde valueSongSerde = new SpecificAvroSerde<>(); + valueSongSerde.configure(serdeConfig, false); + + final SpecificAvroSerde songPlayCountSerde = new SpecificAvroSerde<>(); + songPlayCountSerde.configure(serdeConfig, false); + + // Accept play events that have a duration >= the minimum + final KStream playsBySongId = + playEvents.filter((region, event) -> event.getDuration() >= MIN_CHARTABLE_DURATION) + // repartition based on song id + .map((key, value) -> KeyValue.pair(value.getSongId(), value)); + + // join the plays with song as we will use it later for charting + final KStream songPlays = playsBySongId.leftJoin(songTable, + (value1, song) -> song, + Joined.with(Serdes.Long(), playEventSerde, valueSongSerde)); + + // create a state store to track song play counts + final KTable songPlayCounts = songPlays.groupBy((songId, song) -> song, + Serialized.with(keySongSerde, valueSongSerde)) + .count(Materialized.>as(SONG_PLAY_COUNT_STORE) + .withKeySerde(valueSongSerde) + .withValueSerde(Serdes.Long())); + + final TopFiveSerde topFiveSerde = new TopFiveSerde(); + + // Compute the top five charts for each genre. The results of this computation will continuously update the state + // store "top-five-songs-by-genre", and this state store can then be queried interactively via a REST API (cf. + // MusicPlaysRestService) for the latest charts per genre. + songPlayCounts.groupBy((song, plays) -> + KeyValue.pair(song.getGenre().toLowerCase(), + new SongPlayCount(song.getId(), plays)), + Serialized.with(Serdes.String(), songPlayCountSerde)) + // aggregate into a TopFiveSongs instance that will keep track + // of the current top five for each genre. The data will be available in the + // top-five-songs-genre store + .aggregate(TopFiveSongs::new, + (aggKey, value, aggregate) -> { + aggregate.add(value); + return aggregate; + }, + (aggKey, value, aggregate) -> { + aggregate.remove(value); + return aggregate; + }, + Materialized.>as(TOP_FIVE_SONGS_BY_GENRE_STORE) + .withKeySerde(Serdes.String()) + .withValueSerde(topFiveSerde) + ); + + // Compute the top five chart. The results of this computation will continuously update the state + // store "top-five-songs", and this state store can then be queried interactively via a REST API (cf. + // MusicPlaysRestService) for the latest charts per genre. + songPlayCounts.groupBy((song, plays) -> + KeyValue.pair(TOP_FIVE_KEY, + new SongPlayCount(song.getId(), plays)), + Serialized.with(Serdes.String(), songPlayCountSerde)) + .aggregate(TopFiveSongs::new, + (aggKey, value, aggregate) -> { + aggregate.add(value); + return aggregate; + }, + (aggKey, value, aggregate) -> { + aggregate.remove(value); + return aggregate; + }, + Materialized.>as(TOP_FIVE_SONGS_STORE) + .withKeySerde(Serdes.String()) + .withValueSerde(topFiveSerde) + ); + } + } + + + interface KStreamProcessorX { + + @Input("input") + KStream input(); + + @Input("inputX") + KTable inputX(); + } + + @RestController + public class FooController { + + @RequestMapping("/charts/top-five") + public List greeting(@RequestParam(value="genre") String genre) { + return topFiveSongs(KafkaStreamsInteractiveQuerySample.TOP_FIVE_KEY, KafkaStreamsInteractiveQuerySample.TOP_FIVE_SONGS_STORE); + } + + private List topFiveSongs(final String key, + final String storeName) { + + final ReadOnlyKeyValueStore topFiveStore = + queryableStoreRegistry.getQueryableStoreType(storeName, QueryableStoreTypes.keyValueStore()); + + // Get the value from the store + final TopFiveSongs value = topFiveStore.get(key); + if (value == null) { + throw new IllegalArgumentException(String.format("Unable to find value in %s for key %s", storeName, key)); + } + final List results = new ArrayList<>(); + value.forEach(songPlayCount -> { + final ReadOnlyKeyValueStore songStore = + queryableStoreRegistry.getQueryableStoreType(KafkaStreamsInteractiveQuerySample.ALL_SONGS, QueryableStoreTypes.keyValueStore()); + + final Song song = songStore.get(songPlayCount.getSongId()); + results.add(new SongPlayCountBean(song.getArtist(),song.getAlbum(), song.getName(), + songPlayCount.getPlays())); + }); + return results; + } + } + + /** + * Serde for TopFiveSongs + */ + private static class TopFiveSerde implements Serde { + + @Override + public void configure(final Map map, final boolean b) { + + } + + @Override + public void close() { + + } + + @Override + public Serializer serializer() { + return new Serializer() { + @Override + public void configure(final Map map, final boolean b) { + } + + @Override + public byte[] serialize(final String s, final TopFiveSongs topFiveSongs) { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + final DataOutputStream + dataOutputStream = + new DataOutputStream(out); + try { + for (SongPlayCount songPlayCount : topFiveSongs) { + dataOutputStream.writeLong(songPlayCount.getSongId()); + dataOutputStream.writeLong(songPlayCount.getPlays()); + } + dataOutputStream.flush(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return out.toByteArray(); + } + + @Override + public void close() { + + } + }; + } + + @Override + public Deserializer deserializer() { + return new Deserializer() { + @Override + public void configure(final Map map, final boolean b) { + + } + + @Override + public TopFiveSongs deserialize(final String s, final byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return null; + } + final TopFiveSongs result = new TopFiveSongs(); + + final DataInputStream + dataInputStream = + new DataInputStream(new ByteArrayInputStream(bytes)); + + try { + while(dataInputStream.available() > 0) { + result.add(new SongPlayCount(dataInputStream.readLong(), + dataInputStream.readLong())); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return result; + } + + @Override + public void close() { + + } + }; + } + } + + /** + * Used in aggregations to keep track of the Top five songs + */ + static class TopFiveSongs implements Iterable { + private final Map currentSongs = new HashMap<>(); + private final TreeSet topFive = new TreeSet<>((o1, o2) -> { + final int result = o2.getPlays().compareTo(o1.getPlays()); + if (result != 0) { + return result; + } + return o1.getSongId().compareTo(o2.getSongId()); + }); + + public void add(final SongPlayCount songPlayCount) { + if(currentSongs.containsKey(songPlayCount.getSongId())) { + topFive.remove(currentSongs.remove(songPlayCount.getSongId())); + } + topFive.add(songPlayCount); + currentSongs.put(songPlayCount.getSongId(), songPlayCount); + if (topFive.size() > 5) { + final SongPlayCount last = topFive.last(); + currentSongs.remove(last.getSongId()); + topFive.remove(last); + } + } + + void remove(final SongPlayCount value) { + topFive.remove(value); + currentSongs.remove(value.getSongId()); + } + + + @Override + public Iterator iterator() { + return topFive.iterator(); + } + } + +} diff --git a/kafka-streams/kafka-streams-interactive-query/src/main/java/kafka/streams/interactive/query/Producers.java b/kafka-streams/kafka-streams-interactive-query/src/main/java/kafka/streams/interactive/query/Producers.java new file mode 100644 index 0000000..e6efd72 --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/src/main/java/kafka/streams/interactive/query/Producers.java @@ -0,0 +1,138 @@ +package kafka.streams.interactive.query; + +import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; +import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerializer; +import kafka.streams.interactive.query.avro.PlayEvent; +import kafka.streams.interactive.query.avro.Song; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.LongSerializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; + +import java.util.*; + +/** + * @author Soby Chacko + */ +public class Producers { + + public static void main(String... args) throws Exception { + + final Map serdeConfig = Collections.singletonMap( + AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081"); + final SpecificAvroSerializer playEventSerializer = new SpecificAvroSerializer<>(); + playEventSerializer.configure(serdeConfig, false); + final SpecificAvroSerializer songSerializer = new SpecificAvroSerializer<>(); + songSerializer.configure(serdeConfig, false); + + final List songs = Arrays.asList(new Song(1L, + "Fresh Fruit For Rotting Vegetables", + "Dead Kennedys", + "Chemical Warfare", + "Punk"), + new Song(2L, + "We Are the League", + "Anti-Nowhere League", + "Animal", + "Punk"), + new Song(3L, + "Live In A Dive", + "Subhumans", + "All Gone Dead", + "Punk"), + new Song(4L, + "PSI", + "Wheres The Pope?", + "Fear Of God", + "Punk"), + new Song(5L, + "Totally Exploited", + "The Exploited", + "Punks Not Dead", + "Punk"), + new Song(6L, + "The Audacity Of Hype", + "Jello Biafra And The Guantanamo School Of " + + "Medicine", + "Three Strikes", + "Punk"), + new Song(7L, + "Licensed to Ill", + "The Beastie Boys", + "Fight For Your Right", + "Hip Hop"), + new Song(8L, + "De La Soul Is Dead", + "De La Soul", + "Oodles Of O's", + "Hip Hop"), + new Song(9L, + "Straight Outta Compton", + "N.W.A", + "Gangsta Gangsta", + "Hip Hop"), + new Song(10L, + "Fear Of A Black Planet", + "Public Enemy", + "911 Is A Joke", + "Hip Hop"), + new Song(11L, + "Curtain Call - The Hits", + "Eminem", + "Fack", + "Hip Hop"), + new Song(12L, + "The Calling", + "Hilltop Hoods", + "The Calling", + "Hip Hop") + + ); + + Map props = new HashMap<>(); + props.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081"); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + props.put(ProducerConfig.RETRIES_CONFIG, 0); + props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384); + props.put(ProducerConfig.LINGER_MS_CONFIG, 1); + props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, playEventSerializer.getClass()); + + + Map props1 = new HashMap<>(props); + props1.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class); + props1.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, songSerializer.getClass()); + + DefaultKafkaProducerFactory pf1 = new DefaultKafkaProducerFactory<>(props1); + KafkaTemplate template1 = new KafkaTemplate<>(pf1, true); + template1.setDefaultTopic(KafkaStreamsInteractiveQuerySample.SONG_FEED); + + songs.forEach(song -> { + System.out.println("Writing song information for '" + song.getName() + "' to input topic " + + KafkaStreamsInteractiveQuerySample.SONG_FEED); + template1.sendDefault(song.getId(), song); + }); + + + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(props); + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic(KafkaStreamsInteractiveQuerySample.PLAY_EVENTS); + + + final long duration = 60 * 1000L; + final Random random = new Random(); + + // send a play event every 100 milliseconds + while (true) { + final Song song = songs.get(random.nextInt(songs.size())); + System.out.println("Writing play event for song " + song.getName() + " to input topic " + + KafkaStreamsInteractiveQuerySample.PLAY_EVENTS); + template.sendDefault("uk", new PlayEvent(song.getId(), duration)); + + Thread.sleep(100L); + } + } + +} diff --git a/kafka-streams/kafka-streams-interactive-query/src/main/java/kafka/streams/interactive/query/SongPlayCountBean.java b/kafka-streams/kafka-streams-interactive-query/src/main/java/kafka/streams/interactive/query/SongPlayCountBean.java new file mode 100644 index 0000000..8d71057 --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/src/main/java/kafka/streams/interactive/query/SongPlayCountBean.java @@ -0,0 +1,87 @@ +package kafka.streams.interactive.query; + +import java.util.Objects; + +/** + * @author Soby Chacko + */ +public class SongPlayCountBean { + + private String artist; + private String album; + private String name; + private Long plays; + + public SongPlayCountBean() {} + + public SongPlayCountBean(final String artist, final String album, final String name, final Long + plays) { + + this.artist = artist; + this.album = album; + this.name = name; + this.plays = plays; + } + + public String getArtist() { + return artist; + } + + public void setArtist(final String artist) { + this.artist = artist; + } + + public String getAlbum() { + return album; + } + + public void setAlbum(final String album) { + this.album = album; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public Long getPlays() { + return plays; + } + + public void setPlays(final Long plays) { + this.plays = plays; + } + + @Override + public String toString() { + return "SongPlayCountBean{" + + "artist='" + artist + '\'' + + ", album='" + album + '\'' + + ", name='" + name + '\'' + + ", plays=" + plays + + '}'; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final SongPlayCountBean that = (SongPlayCountBean) o; + return Objects.equals(artist, that.artist) && + Objects.equals(album, that.album) && + Objects.equals(name, that.name) && + Objects.equals(plays, that.plays); + } + + @Override + public int hashCode() { + return Objects.hash(artist, album, name, plays); + } +} diff --git a/kafka-streams/kafka-streams-interactive-query/src/main/resources/application.yml b/kafka-streams/kafka-streams-interactive-query/src/main/resources/application.yml new file mode 100644 index 0000000..bdfd8a6 --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/src/main/resources/application.yml @@ -0,0 +1,25 @@ +spring.cloud.stream.bindings.input: + destination: play-events + consumer: + useNativeDecoding: true + headerMode: raw +spring.cloud.stream.bindings.inputX: + destination: song-feed + consumer: + useNativeDecoding: true + headerMode: raw +spring.cloud.stream.kafka.streams.bindings.input: + consumer: + keySerde: org.apache.kafka.common.serialization.Serdes$StringSerde + valueSerde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde +spring.cloud.stream.kafka.streams.bindings.inputX: + consumer: + keySerde: org.apache.kafka.common.serialization.Serdes$LongSerde + valueSerde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde + materializedAs: all-songs +spring.cloud.stream.kafka.streams.binder: + brokers: localhost #192.168.99.100 + zkNodes: localhost #192.168.99.100 + configuration: + schema.registry.url: http://localhost:8081 + commit.interval.ms: 1000 \ No newline at end of file diff --git a/kafka-streams/kafka-streams-interactive-query/src/main/resources/avro/kafka/streams/interactive/query/playevent.avsc b/kafka-streams/kafka-streams-interactive-query/src/main/resources/avro/kafka/streams/interactive/query/playevent.avsc new file mode 100644 index 0000000..ca904b3 --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/src/main/resources/avro/kafka/streams/interactive/query/playevent.avsc @@ -0,0 +1,8 @@ +{"namespace": "kafka.streams.interactive.query.avro", + "type": "record", + "name": "PlayEvent", + "fields": [ + {"name": "song_id", "type": "long"}, + {"name": "duration", "type": "long"} + ] +} \ No newline at end of file diff --git a/kafka-streams/kafka-streams-interactive-query/src/main/resources/avro/kafka/streams/interactive/query/song.avsc b/kafka-streams/kafka-streams-interactive-query/src/main/resources/avro/kafka/streams/interactive/query/song.avsc new file mode 100644 index 0000000..2a05f2f --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/src/main/resources/avro/kafka/streams/interactive/query/song.avsc @@ -0,0 +1,11 @@ +{"namespace": "kafka.streams.interactive.query.avro", + "type": "record", + "name": "Song", + "fields": [ + {"name": "id", "type": "long"}, + {"name": "album", "type": "string"}, + {"name": "artist", "type": "string"}, + {"name": "name", "type": "string"}, + {"name": "genre", "type": "string"} + ] +} \ No newline at end of file diff --git a/kafka-streams/kafka-streams-interactive-query/src/main/resources/avro/kafka/streams/interactive/query/songplaycount.avsc b/kafka-streams/kafka-streams-interactive-query/src/main/resources/avro/kafka/streams/interactive/query/songplaycount.avsc new file mode 100644 index 0000000..6017ce7 --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/src/main/resources/avro/kafka/streams/interactive/query/songplaycount.avsc @@ -0,0 +1,8 @@ +{"namespace": "kafka.streams.interactive.query.avro", + "type": "record", + "name": "SongPlayCount", + "fields": [ + {"name": "song_id", "type": "long"}, + {"name": "plays", "type": "long"} + ] +} \ No newline at end of file diff --git a/kafka-streams/kafka-streams-interactive-query/src/main/resources/logback.xml b/kafka-streams/kafka-streams-interactive-query/src/main/resources/logback.xml new file mode 100644 index 0000000..870ac9e --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/src/main/resources/logback.xml @@ -0,0 +1,12 @@ + + + + + %d{ISO8601} %5p %t %c{2}:%L - %m%n + + + + + + + \ No newline at end of file diff --git a/kafka-streams/kafka-streams-interactive-query/src/test/java/kafka/streams/interactive/query/KafkaStreamsInteractiveQueryTests.java b/kafka-streams/kafka-streams-interactive-query/src/test/java/kafka/streams/interactive/query/KafkaStreamsInteractiveQueryTests.java new file mode 100644 index 0000000..1d0b41a --- /dev/null +++ b/kafka-streams/kafka-streams-interactive-query/src/test/java/kafka/streams/interactive/query/KafkaStreamsInteractiveQueryTests.java @@ -0,0 +1,18 @@ +package kafka.streams.interactive.query; + +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +public class KafkaStreamsInteractiveQueryTests { + + @Test + @Ignore + public void contextLoads() { + } + +} diff --git a/kafka-streams/kafka-streams-message-channel/.gitignore b/kafka-streams/kafka-streams-message-channel/.gitignore new file mode 100644 index 0000000..2af7cef --- /dev/null +++ b/kafka-streams/kafka-streams-message-channel/.gitignore @@ -0,0 +1,24 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +nbproject/private/ +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ \ No newline at end of file diff --git a/kafka-streams/kafka-streams-message-channel/.mvn/wrapper/maven-wrapper.jar b/kafka-streams/kafka-streams-message-channel/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000..9cc84ea Binary files /dev/null and b/kafka-streams/kafka-streams-message-channel/.mvn/wrapper/maven-wrapper.jar differ diff --git a/kafka-streams/kafka-streams-message-channel/.mvn/wrapper/maven-wrapper.properties b/kafka-streams/kafka-streams-message-channel/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..c315043 --- /dev/null +++ b/kafka-streams/kafka-streams-message-channel/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1 @@ +distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.0/apache-maven-3.5.0-bin.zip diff --git a/kafka-streams/kafka-streams-message-channel/README.adoc b/kafka-streams/kafka-streams-message-channel/README.adoc new file mode 100644 index 0000000..494b588 --- /dev/null +++ b/kafka-streams/kafka-streams-message-channel/README.adoc @@ -0,0 +1,47 @@ +== What is this app? + +This is an example of a Spring Cloud Stream processor using Kafka Streams support. + +The example is based on the word count application from the https://github.com/confluentinc/examples/blob/3.2.x/kafka-streams/src/main/java/io/confluent/examples/streams/WordCountLambdaExample.java[reference documentation]. +The application has two inputs one as a KStream and another one with the binding for a message channel. + +==== Starting Kafka in a docker container + +* Skip steps 1-3 if you already have a non-Docker Kafka environment. + +1. Go to the docker directory in this repo and invoke the command `docker-compose up -d`. +2. Ensure that in the docker directory and then invoke the script `start-kafka-shell.sh` +3. cd $KAFKA_HOME +4. Start the console producer: + +Assuming that you are running kafka on a docker container on mac osx. Change the zookeeper IP address accordingly otherwise. + +`bin/kafka-console-producer.sh --broker-list 192.168.99.100:9092 --topic words` +5. Start the console consumer: + +Assuming that you are running kafka on a docker container on mac osx. Change the zookeeper IP address accordingly otherwise. + +`bin/kafka-console-consumer.sh --bootstrap-server 192.168.99.100:9092 --topic counts` + +=== Running the app: + +Go to the root of the repository and do: `./mvnw clean package` + +`java -jar target/kstream-word-count-0.0.1-SNAPSHOT.jar` + +* By default we use the docker container IP (mac osx specific) in the `application.yml` for Kafka broker and zookeeper. +Change it in `application.yml` (which requires a rebuild) or pass them as runtime arguments as below. + +`spring.cloud.stream.kstream.binder.brokers=` + +`spring.cloud.stream.kstream.binder.zkNodes=` + +Enter some text in the console producer and watch the output in the console consumer. + +Also watch the console for logging statements from the regular sink StreamListener method. + +The default time window is configured for 5 seconds and you can change that using the following property. + +`spring.cloud.stream.kstream.timeWindow.length` (value is expressed in milliseconds) + +In order to switch to a hopping window, you can use the `spring.cloud.stream.kstream.timeWindow.advnceBy` (value in milliseconds). +This will create an overlapped hopping windows depending on the value you provide. + +Here is an example with 2 overlapping windows (window length of 10 seconds and a hop (advance) by 5 seconds: + +`java -jar target/kstream-word-count-0.0.1-SNAPSHOT.jar --spring.cloud.stream.kstream.timeWindow.length=10000 --spring.cloud.stream.kstream.timeWindow.advnceBy=5000` \ No newline at end of file diff --git a/kafka-streams/kafka-streams-message-channel/docker/docker-compose.yml b/kafka-streams/kafka-streams-message-channel/docker/docker-compose.yml new file mode 100644 index 0000000..ecf41fb --- /dev/null +++ b/kafka-streams/kafka-streams-message-channel/docker/docker-compose.yml @@ -0,0 +1,14 @@ +version: '2' +services: + zookeeper: + image: wurstmeister/zookeeper + ports: + - "2181:2181" + kafka: + image: wurstmeister/kafka + ports: + - "9092:9092" + environment: + KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100 + KAFKA_ADVERTISED_PORT: 9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 diff --git a/kafka-streams/kafka-streams-message-channel/docker/start-kafka-shell.sh b/kafka-streams/kafka-streams-message-channel/docker/start-kafka-shell.sh new file mode 100755 index 0000000..62663e4 --- /dev/null +++ b/kafka-streams/kafka-streams-message-channel/docker/start-kafka-shell.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e HOST_IP=$1 -e ZK=$2 -i -t wurstmeister/kafka /bin/bash diff --git a/kafka-streams/kafka-streams-message-channel/mvnw b/kafka-streams/kafka-streams-message-channel/mvnw new file mode 100755 index 0000000..5bf251c --- /dev/null +++ b/kafka-streams/kafka-streams-message-channel/mvnw @@ -0,0 +1,225 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Migwn, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +echo $MAVEN_PROJECTBASEDIR +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/kafka-streams/kafka-streams-message-channel/mvnw.cmd b/kafka-streams/kafka-streams-message-channel/mvnw.cmd new file mode 100644 index 0000000..019bd74 --- /dev/null +++ b/kafka-streams/kafka-streams-message-channel/mvnw.cmd @@ -0,0 +1,143 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" + +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/kafka-streams/kafka-streams-message-channel/pom.xml b/kafka-streams/kafka-streams-message-channel/pom.xml new file mode 100644 index 0000000..a469828 --- /dev/null +++ b/kafka-streams/kafka-streams-message-channel/pom.xml @@ -0,0 +1,88 @@ + + + 4.0.0 + + kafka.streams.message.channel + kafka-streams-message-channel + 0.0.1-SNAPSHOT + jar + + kafka-streams-message-channel + Demo project for Spring Boot + + + org.springframework.boot + spring-boot-starter-parent + 2.0.0.BUILD-SNAPSHOT + + + + + UTF-8 + UTF-8 + 1.8 + Finchley.BUILD-SNAPSHOT + + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.kafka + spring-kafka + 2.1.3.BUILD-SNAPSHOT + + + org.springframework.cloud + spring-cloud-stream-binder-kafka-streams + 2.0.0.BUILD-SNAPSHOT + + + org.springframework.cloud + spring-cloud-stream-binder-kafka + 2.0.0.BUILD-SNAPSHOT + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.springframework.cloud + spring-cloud-dependencies + ${spring-cloud.version} + pom + import + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + + spring-milestones + Spring Milestones + http://repo.spring.io/libs-milestone-local + + false + + + + + diff --git a/kafka-streams/kafka-streams-message-channel/src/main/java/kafka/streams/message/channel/KafkaStreamsWordCountApplication.java b/kafka-streams/kafka-streams-message-channel/src/main/java/kafka/streams/message/channel/KafkaStreamsWordCountApplication.java new file mode 100644 index 0000000..ed5c34c --- /dev/null +++ b/kafka-streams/kafka-streams-message-channel/src/main/java/kafka/streams/message/channel/KafkaStreamsWordCountApplication.java @@ -0,0 +1,123 @@ +package kafka.streams.message.channel; + +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Serialized; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.annotation.Output; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.messaging.SubscribableChannel; +import org.springframework.messaging.handler.annotation.SendTo; + +import java.util.Arrays; +import java.util.Date; + +@SpringBootApplication +public class KafkaStreamsWordCountApplication { + + public static void main(String[] args) { + SpringApplication.run(KafkaStreamsWordCountApplication.class, args); + } + + @EnableBinding(MultipleProcessor.class) + @EnableAutoConfiguration + public static class WordCountProcessorApplication { + + @Autowired + private TimeWindows timeWindows; + + @StreamListener("binding2") + @SendTo("singleOutput") + public KStream process(KStream input) { + + return input + .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) + .map((key, value) -> new KeyValue<>(value, value)) + .groupByKey(Serialized.with(Serdes.String(), Serdes.String())) + .windowedBy(timeWindows) + .count(Materialized.as("WordCounts-1")) + .toStream() + .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))); + } + + @StreamListener("binding1") + public void sink(String input) { + System.out.println("FOOBAR -- " + input); + } + + } + + interface MultipleProcessor { + + String BINDING_1 = "binding1"; + String BINDING_2 = "binding2"; + String OUTPUT = "singleOutput"; + + @Input(BINDING_1) + SubscribableChannel binding1(); + + @Input(BINDING_2) + KStream binding2(); + + @Output(OUTPUT) + KStream singleOutput(); + } + + static class WordCount { + + private String word; + + private long count; + + private Date start; + + private Date end; + + WordCount(String word, long count, Date start, Date end) { + this.word = word; + this.count = count; + this.start = start; + this.end = end; + } + + public String getWord() { + return word; + } + + public void setWord(String word) { + this.word = word; + } + + public long getCount() { + return count; + } + + public void setCount(long count) { + this.count = count; + } + + public Date getStart() { + return start; + } + + public void setStart(Date start) { + this.start = start; + } + + public Date getEnd() { + return end; + } + + public void setEnd(Date end) { + this.end = end; + } + } +} diff --git a/kafka-streams/kafka-streams-message-channel/src/main/resources/application.yml b/kafka-streams/kafka-streams-message-channel/src/main/resources/application.yml new file mode 100644 index 0000000..3db7566 --- /dev/null +++ b/kafka-streams/kafka-streams-message-channel/src/main/resources/application.yml @@ -0,0 +1,24 @@ +spring.cloud.stream.bindings.singleOutput.contentType: application/json +spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms: 1000 +spring.cloud.stream.kafka.streams.binder.configuration: + default.key.serde: org.apache.kafka.common.serialization.Serdes$StringSerde + default.value.serde: org.apache.kafka.common.serialization.Serdes$StringSerde +spring.cloud.stream.bindings.singleOutput: + destination: counts + producer: + headerMode: raw + #useNativeEncoding: true +spring.cloud.stream.bindings.binding2: + destination: words + consumer: + headerMode: raw +spring.cloud.stream.bindings.binding1: + destination: words + consumer: + headerMode: raw +spring.cloud.stream.kafka.streams.binder: + brokers: localhost #192.168.99.100 + zkNodes: localhost #192.168.99.100 + + + diff --git a/kafka-streams/kafka-streams-message-channel/src/main/resources/logback.xml b/kafka-streams/kafka-streams-message-channel/src/main/resources/logback.xml new file mode 100644 index 0000000..870ac9e --- /dev/null +++ b/kafka-streams/kafka-streams-message-channel/src/main/resources/logback.xml @@ -0,0 +1,12 @@ + + + + + %d{ISO8601} %5p %t %c{2}:%L - %m%n + + + + + + + \ No newline at end of file diff --git a/kafka-streams/kafka-streams-message-channel/src/test/java/kafka/streams/message/channel/KafkaStreamsWordCountApplicationTests.java b/kafka-streams/kafka-streams-message-channel/src/test/java/kafka/streams/message/channel/KafkaStreamsWordCountApplicationTests.java new file mode 100644 index 0000000..12a0160 --- /dev/null +++ b/kafka-streams/kafka-streams-message-channel/src/test/java/kafka/streams/message/channel/KafkaStreamsWordCountApplicationTests.java @@ -0,0 +1,18 @@ +package kafka.streams.message.channel; + +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +public class KafkaStreamsWordCountApplicationTests { + + @Test + @Ignore + public void contextLoads() { + } + +} diff --git a/kafka-streams/pom.xml b/kafka-streams/pom.xml index ed1803d..9c42dcf 100644 --- a/kafka-streams/pom.xml +++ b/kafka-streams/pom.xml @@ -5,6 +5,12 @@ spring-cloud-stream-kafka-stream-samples pom + + org.springframework.cloud + spring-cloud-stream-samples + 1.2.0.BUILD-SNAPSHOT + + spring-cloud-stream-kafka-stream-samples Parent Project for Kafka Streams Samples @@ -13,5 +19,7 @@ kafka-streams-branching-sample kafka-streams-dlq-sample kafka-streams-table-join + kafka-streams-interactive-query + kafka-streams-message-channel - \ No newline at end of file + diff --git a/pom.xml b/pom.xml index ff52382..e77d353 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,6 @@ kinesis-produce-consume kstream testing -