Refactoring samples

Restructuring the samples repository
Add more docker support
Add acceptance tests for the apps
Adding sensor average processor sample
Remove aggregate samples

kafka-streams-samples/kafka-streams-interactive-query-advanced/.gitignore

@@ -0,0 +1,24 @@
+target/
+!.mvn/wrapper/maven-wrapper.jar
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+nbproject/private/
+build/
+nbbuild/
+dist/
+nbdist/
+.nb-gradle/

kafka-streams-samples/kafka-streams-interactive-query-advanced/.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1 @@
+distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.0/apache-maven-3.5.0-bin.zip

kafka-streams-samples/kafka-streams-interactive-query-advanced/README.adoc

@@ -0,0 +1,26 @@
+== What is this app?
+
+This is an example of a Spring Cloud Stream processor using Kafka Streams support.
+
+This example is a Spring Cloud Stream adaptation of this Kafka Streams sample: https://github.com/confluentinc/kafka-streams-examples/tree/4.0.0-post/src/main/java/io/confluent/examples/streams/interactivequeries/kafkamusic
+
+This sample demonstrates the concept of interactive queries in kafka streams.
+There is a REST service provided as part of the application that can be used to query the store interactively.
+
+=== Running the app:
+
+1. `docker-compose up -d`
+
+2. Start the confluent schema registry: The following command is based on the confluent platform.
+
+`./bin/schema-registry-start ./etc/schema-registry/schema-registry.properties`
+
+3. Go to the root of the repository and do: `./mvnw clean package`
+
+4. `java -jar target/kafka-streams-interactive-query-0.0.1-SNAPSHOT.jar`
+
+5. Run the stand-alone `Producers` application to generate data and start the processing.
+Keep it running for a while.
+
+6. Go to the URL: http://localhost:8080/charts/top-five?genre=Punk
+keep refreshing the URL and you will see the song play count information changes.

kafka-streams-samples/kafka-streams-interactive-query-advanced/docker-compose.yml

@@ -0,0 +1,19 @@
+version: '3'
+services:
+    kafka:
+      image: wurstmeister/kafka
+      container_name: kafka-iq-advanced
+      ports:
+        - "9092:9092"
+      environment:
+        - KAFKA_ADVERTISED_HOST_NAME=127.0.0.1
+        - KAFKA_ADVERTISED_PORT=9092
+        - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      depends_on:
+        - zookeeper
+    zookeeper:
+      image: wurstmeister/zookeeper
+      ports:
+        - "2181:2181"
+      environment:
+        - KAFKA_ADVERTISED_HOST_NAME=zookeeper

kafka-streams-samples/kafka-streams-interactive-query-advanced/mvnw

@@ -0,0 +1,225 @@
+#!/bin/sh
+# ----------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Maven2 Start Up Batch script
+#
+# Required ENV vars:
+# ------------------
+#   JAVA_HOME - location of a JDK home dir
+#
+# Optional ENV vars
+# -----------------
+#   M2_HOME - location of maven2's installed home dir
+#   MAVEN_OPTS - parameters passed to the Java VM when running Maven
+#     e.g. to debug Maven itself, use
+#       set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+#   MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+# ----------------------------------------------------------------------------
+
+if [ -z "$MAVEN_SKIP_RC" ] ; then
+
+  if [ -f /etc/mavenrc ] ; then
+    . /etc/mavenrc
+  fi
+
+  if [ -f "$HOME/.mavenrc" ] ; then
+    . "$HOME/.mavenrc"
+  fi
+
+fi
+
+# OS specific support.  $var _must_ be set to either true or false.
+cygwin=false;
+darwin=false;
+mingw=false
+case "`uname`" in
+  CYGWIN*) cygwin=true ;;
+  MINGW*) mingw=true;;
+  Darwin*) darwin=true
+    # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
+    # See https://developer.apple.com/library/mac/qa/qa1170/_index.html
+    if [ -z "$JAVA_HOME" ]; then
+      if [ -x "/usr/libexec/java_home" ]; then
+        export JAVA_HOME="`/usr/libexec/java_home`"
+      else
+        export JAVA_HOME="/Library/Java/Home"
+      fi
+    fi
+    ;;
+esac
+
+if [ -z "$JAVA_HOME" ] ; then
+  if [ -r /etc/gentoo-release ] ; then
+    JAVA_HOME=`java-config --jre-home`
+  fi
+fi
+
+if [ -z "$M2_HOME" ] ; then
+  ## resolve links - $0 may be a link to maven's home
+  PRG="$0"
+
+  # need this for relative symlinks
+  while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+      PRG="$link"
+    else
+      PRG="`dirname "$PRG"`/$link"
+    fi
+  done
+
+  saveddir=`pwd`
+
+  M2_HOME=`dirname "$PRG"`/..
+
+  # make it fully qualified
+  M2_HOME=`cd "$M2_HOME" && pwd`
+
+  cd "$saveddir"
+  # echo Using m2 at $M2_HOME
+fi
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin ; then
+  [ -n "$M2_HOME" ] &&
+    M2_HOME=`cygpath --unix "$M2_HOME"`
+  [ -n "$JAVA_HOME" ] &&
+    JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+  [ -n "$CLASSPATH" ] &&
+    CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
+fi
+
+# For Migwn, ensure paths are in UNIX format before anything is touched
+if $mingw ; then
+  [ -n "$M2_HOME" ] &&
+    M2_HOME="`(cd "$M2_HOME"; pwd)`"
+  [ -n "$JAVA_HOME" ] &&
+    JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
+  # TODO classpath?
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+  javaExecutable="`which javac`"
+  if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
+    # readlink(1) is not available as standard on Solaris 10.
+    readLink=`which readlink`
+    if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
+      if $darwin ; then
+        javaHome="`dirname \"$javaExecutable\"`"
+        javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
+      else
+        javaExecutable="`readlink -f \"$javaExecutable\"`"
+      fi
+      javaHome="`dirname \"$javaExecutable\"`"
+      javaHome=`expr "$javaHome" : '\(.*\)/bin'`
+      JAVA_HOME="$javaHome"
+      export JAVA_HOME
+    fi
+  fi
+fi
+
+if [ -z "$JAVACMD" ] ; then
+  if [ -n "$JAVA_HOME"  ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+      # IBM's JDK on AIX uses strange locations for the executables
+      JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+      JAVACMD="$JAVA_HOME/bin/java"
+    fi
+  else
+    JAVACMD="`which java`"
+  fi
+fi
+
+if [ ! -x "$JAVACMD" ] ; then
+  echo "Error: JAVA_HOME is not defined correctly." >&2
+  echo "  We cannot execute $JAVACMD" >&2
+  exit 1
+fi
+
+if [ -z "$JAVA_HOME" ] ; then
+  echo "Warning: JAVA_HOME environment variable is not set."
+fi
+
+CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
+
+# traverses directory structure from process work directory to filesystem root
+# first directory with .mvn subdirectory is considered project base directory
+find_maven_basedir() {
+
+  if [ -z "$1" ]
+  then
+    echo "Path not specified to find_maven_basedir"
+    return 1
+  fi
+
+  basedir="$1"
+  wdir="$1"
+  while [ "$wdir" != '/' ] ; do
+    if [ -d "$wdir"/.mvn ] ; then
+      basedir=$wdir
+      break
+    fi
+    # workaround for JBEAP-8937 (on Solaris 10/Sparc)
+    if [ -d "${wdir}" ]; then
+      wdir=`cd "$wdir/.."; pwd`
+    fi
+    # end of workaround
+  done
+  echo "${basedir}"
+}
+
+# concatenates all lines of a file
+concat_lines() {
+  if [ -f "$1" ]; then
+    echo "$(tr -s '\n' ' ' < "$1")"
+  fi
+}
+
+BASE_DIR=`find_maven_basedir "$(pwd)"`
+if [ -z "$BASE_DIR" ]; then
+  exit 1;
+fi
+
+export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
+echo $MAVEN_PROJECTBASEDIR
+MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin; then
+  [ -n "$M2_HOME" ] &&
+    M2_HOME=`cygpath --path --windows "$M2_HOME"`
+  [ -n "$JAVA_HOME" ] &&
+    JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
+  [ -n "$CLASSPATH" ] &&
+    CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
+  [ -n "$MAVEN_PROJECTBASEDIR" ] &&
+    MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
+fi
+
+WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+exec "$JAVACMD" \
+  $MAVEN_OPTS \
+  -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
+  "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
+  ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"

kafka-streams-samples/kafka-streams-interactive-query-advanced/mvnw.cmd

@@ -0,0 +1,143 @@
+@REM ----------------------------------------------------------------------------
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements.  See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership.  The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License.  You may obtain a copy of the License at
+@REM
+@REM    http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied.  See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM ----------------------------------------------------------------------------
+
+@REM ----------------------------------------------------------------------------
+@REM Maven2 Start Up Batch script
+@REM
+@REM Required ENV vars:
+@REM JAVA_HOME - location of a JDK home dir
+@REM
+@REM Optional ENV vars
+@REM M2_HOME - location of maven2's installed home dir
+@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
+@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending
+@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
+@REM     e.g. to debug Maven itself, use
+@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+@REM ----------------------------------------------------------------------------
+
+@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
+@echo off
+@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on'
+@if "%MAVEN_BATCH_ECHO%" == "on"  echo %MAVEN_BATCH_ECHO%
+
+@REM set %HOME% to equivalent of $HOME
+if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
+
+@REM Execute a user defined script before this one
+if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
+@REM check for pre script, once with legacy .bat ending and once with .cmd ending
+if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
+if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
+:skipRcPre
+
+@setlocal
+
+set ERROR_CODE=0
+
+@REM To isolate internal variables from possible post scripts, we use another setlocal
+@setlocal
+
+@REM ==== START VALIDATION ====
+if not "%JAVA_HOME%" == "" goto OkJHome
+
+echo.
+echo Error: JAVA_HOME not found in your environment. >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+:OkJHome
+if exist "%JAVA_HOME%\bin\java.exe" goto init
+
+echo.
+echo Error: JAVA_HOME is set to an invalid directory. >&2
+echo JAVA_HOME = "%JAVA_HOME%" >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+@REM ==== END VALIDATION ====
+
+:init
+
+@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
+@REM Fallback to current working directory if not found.
+
+set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
+IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
+
+set EXEC_DIR=%CD%
+set WDIR=%EXEC_DIR%
+:findBaseDir
+IF EXIST "%WDIR%"\.mvn goto baseDirFound
+cd ..
+IF "%WDIR%"=="%CD%" goto baseDirNotFound
+set WDIR=%CD%
+goto findBaseDir
+
+:baseDirFound
+set MAVEN_PROJECTBASEDIR=%WDIR%
+cd "%EXEC_DIR%"
+goto endDetectBaseDir
+
+:baseDirNotFound
+set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
+cd "%EXEC_DIR%"
+
+:endDetectBaseDir
+
+IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
+
+@setlocal EnableExtensions EnableDelayedExpansion
+for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
+@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
+
+:endReadAdditionalConfig
+
+SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
+
+set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
+set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
+if ERRORLEVEL 1 goto error
+goto end
+
+:error
+set ERROR_CODE=1
+
+:end
+@endlocal & set ERROR_CODE=%ERROR_CODE%
+
+if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
+@REM check for post script, once with legacy .bat ending and once with .cmd ending
+if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
+if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
+:skipRcPost
+
+@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
+if "%MAVEN_BATCH_PAUSE%" == "on" pause
+
+if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
+
+exit /B %ERROR_CODE%

kafka-streams-samples/kafka-streams-interactive-query-advanced/pom.xml

@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<artifactId>kafka-streams-interactive-query-advanced</artifactId>
+	<version>0.0.1-SNAPSHOT</version>
+	<packaging>jar</packaging>
+
+	<name>kafka-streams-interactive-query-advanced</name>
+	<description>Spring Cloud Stream sample for KStream interactive queries</description>
+
+	<parent>
+		<groupId>spring.cloud.stream.samples</groupId>
+		<artifactId>spring-cloud-stream-samples-parent</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+		<relativePath>../..</relativePath>
+	</parent>
+
+	<properties>
+		<confluent.version>4.0.0</confluent.version>
+		<avro.version>1.8.2</avro.version>
+	</properties>
+
+	<dependencies>
+		<dependency>
+			<groupId>io.confluent</groupId>
+			<artifactId>kafka-streams-avro-serde</artifactId>
+			<version>${confluent.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+            </exclusions>
+		</dependency>
+		<dependency>
+			<groupId>io.confluent</groupId>
+			<artifactId>kafka-avro-serializer</artifactId>
+			<version>${confluent.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>io.confluent</groupId>
+			<artifactId>kafka-schema-registry-client</artifactId>
+			<version>${confluent.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.cloud</groupId>
+			<artifactId>spring-cloud-stream-binder-kafka-streams</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-test</artifactId>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.avro</groupId>
+				<artifactId>avro-maven-plugin</artifactId>
+				<version>${avro.version}</version>
+				<executions>
+					<execution>
+						<phase>generate-sources</phase>
+						<goals>
+							<goal>schema</goal>
+						</goals>
+						<configuration>
+							<sourceDirectory>src/main/resources/avro/kafka/streams/interactive/query</sourceDirectory>
+							<outputDirectory>${project.build.directory}/generated-sources</outputDirectory>
+							<stringType>String</stringType>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+			<plugin>
+				<groupId>org.springframework.boot</groupId>
+				<artifactId>spring-boot-maven-plugin</artifactId>
+			</plugin>
+		</plugins>
+
+	</build>
+
+	<repositories>
+		<repository>
+			<id>confluent</id>
+			<url>http://packages.confluent.io/maven/</url>
+		</repository>
+	</repositories>
+
+
+
+</project>

kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/KafkaStreamsInteractiveQuerySample.java

@@ -0,0 +1,326 @@
+/*
+ * Copyright 2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.streams.interactive.query;
+
+import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig;
+import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde;
+import kafka.streams.interactive.query.avro.PlayEvent;
+import kafka.streams.interactive.query.avro.Song;
+import kafka.streams.interactive.query.avro.SongPlayCount;
+import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.Serializer;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.kstream.*;
+import org.apache.kafka.streams.state.KeyValueStore;
+import org.apache.kafka.streams.state.QueryableStoreTypes;
+import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.cloud.stream.annotation.EnableBinding;
+import org.springframework.cloud.stream.annotation.Input;
+import org.springframework.cloud.stream.annotation.StreamListener;
+import org.springframework.cloud.stream.binder.kafka.streams.QueryableStoreRegistry;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.io.*;
+import java.util.*;
+
+@SpringBootApplication
+public class KafkaStreamsInteractiveQuerySample {
+	static final String PLAY_EVENTS = "play-events";
+	static final String SONG_FEED = "song-feed";
+	static final String TOP_FIVE_KEY = "all";
+	static final String TOP_FIVE_SONGS_STORE = "top-five-songs";
+	static final String ALL_SONGS = "all-songs";
+
+	@Autowired
+	private QueryableStoreRegistry queryableStoreRegistry;
+
+	public static void main(String[] args) {
+		SpringApplication.run(KafkaStreamsInteractiveQuerySample.class, args);
+	}
+
+	@EnableBinding(KStreamProcessorX.class)
+	@EnableAutoConfiguration
+	public static class KStreamMusicSampleApplication {
+
+		private static final Long MIN_CHARTABLE_DURATION = 30 * 1000L;
+		private static final String SONG_PLAY_COUNT_STORE = "song-play-count";
+
+		static final String TOP_FIVE_SONGS_BY_GENRE_STORE = "top-five-songs-by-genre";
+		static final String TOP_FIVE_KEY = "all";
+
+		@StreamListener
+		public void process(@Input("input") KStream<String, PlayEvent> playEvents,
+							@Input("inputX") KTable<Long, Song> songTable) {
+
+			// create and configure the SpecificAvroSerdes required in this example
+			final Map<String, String> serdeConfig = Collections.singletonMap(
+					AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081");
+
+			final SpecificAvroSerde<PlayEvent> playEventSerde = new SpecificAvroSerde<>();
+			playEventSerde.configure(serdeConfig, false);
+
+			final SpecificAvroSerde<Song> keySongSerde = new SpecificAvroSerde<>();
+			keySongSerde.configure(serdeConfig, true);
+
+			final SpecificAvroSerde<Song> valueSongSerde = new SpecificAvroSerde<>();
+			valueSongSerde.configure(serdeConfig, false);
+
+			final SpecificAvroSerde<SongPlayCount> songPlayCountSerde = new SpecificAvroSerde<>();
+			songPlayCountSerde.configure(serdeConfig, false);
+
+			// Accept play events that have a duration >= the minimum
+			final KStream<Long, PlayEvent> playsBySongId =
+					playEvents.filter((region, event) -> event.getDuration() >= MIN_CHARTABLE_DURATION)
+							// repartition based on song id
+							.map((key, value) -> KeyValue.pair(value.getSongId(), value));
+
+			// join the plays with song as we will use it later for charting
+			final KStream<Long, Song> songPlays = playsBySongId.leftJoin(songTable,
+					(value1, song) -> song,
+					Joined.with(Serdes.Long(), playEventSerde, valueSongSerde));
+
+			// create a state store to track song play counts
+			final KTable<Song, Long> songPlayCounts = songPlays.groupBy((songId, song) -> song,
+					Serialized.with(keySongSerde, valueSongSerde))
+					.count(Materialized.<Song, Long, KeyValueStore<Bytes, byte[]>>as(SONG_PLAY_COUNT_STORE)
+							.withKeySerde(valueSongSerde)
+							.withValueSerde(Serdes.Long()));
+
+			final TopFiveSerde topFiveSerde = new TopFiveSerde();
+
+			// Compute the top five charts for each genre. The results of this computation will continuously update the state
+			// store "top-five-songs-by-genre", and this state store can then be queried interactively via a REST API (cf.
+			// MusicPlaysRestService) for the latest charts per genre.
+			songPlayCounts.groupBy((song, plays) ->
+							KeyValue.pair(song.getGenre().toLowerCase(),
+									new SongPlayCount(song.getId(), plays)),
+					Serialized.with(Serdes.String(), songPlayCountSerde))
+					// aggregate into a TopFiveSongs instance that will keep track
+					// of the current top five for each genre. The data will be available in the
+					// top-five-songs-genre store
+					.aggregate(TopFiveSongs::new,
+							(aggKey, value, aggregate) -> {
+								aggregate.add(value);
+								return aggregate;
+							},
+							(aggKey, value, aggregate) -> {
+								aggregate.remove(value);
+								return aggregate;
+							},
+							Materialized.<String, TopFiveSongs, KeyValueStore<Bytes, byte[]>>as(TOP_FIVE_SONGS_BY_GENRE_STORE)
+									.withKeySerde(Serdes.String())
+									.withValueSerde(topFiveSerde)
+					);
+
+			// Compute the top five chart. The results of this computation will continuously update the state
+			// store "top-five-songs", and this state store can then be queried interactively via a REST API (cf.
+			// MusicPlaysRestService) for the latest charts per genre.
+			songPlayCounts.groupBy((song, plays) ->
+							KeyValue.pair(TOP_FIVE_KEY,
+									new SongPlayCount(song.getId(), plays)),
+					Serialized.with(Serdes.String(), songPlayCountSerde))
+					.aggregate(TopFiveSongs::new,
+							(aggKey, value, aggregate) -> {
+								aggregate.add(value);
+								return aggregate;
+							},
+							(aggKey, value, aggregate) -> {
+								aggregate.remove(value);
+								return aggregate;
+							},
+							Materialized.<String, TopFiveSongs, KeyValueStore<Bytes, byte[]>>as(TOP_FIVE_SONGS_STORE)
+									.withKeySerde(Serdes.String())
+									.withValueSerde(topFiveSerde)
+					);
+		}
+	}
+
+
+	interface KStreamProcessorX {
+
+		@Input("input")
+		KStream<?, ?> input();
+
+		@Input("inputX")
+		KTable<?, ?> inputX();
+	}
+
+	@RestController
+	public class FooController {
+
+		@RequestMapping("/charts/top-five")
+		public List<SongPlayCountBean> greeting(@RequestParam(value="genre") String genre) {
+			return topFiveSongs(KafkaStreamsInteractiveQuerySample.TOP_FIVE_KEY, KafkaStreamsInteractiveQuerySample.TOP_FIVE_SONGS_STORE);
+		}
+
+		private List<SongPlayCountBean> topFiveSongs(final String key,
+													 final String storeName) {
+
+			final ReadOnlyKeyValueStore<String, TopFiveSongs> topFiveStore =
+					queryableStoreRegistry.getQueryableStoreType(storeName, QueryableStoreTypes.<String, TopFiveSongs>keyValueStore());
+
+			// Get the value from the store
+			final TopFiveSongs value = topFiveStore.get(key);
+			if (value == null) {
+				throw new IllegalArgumentException(String.format("Unable to find value in %s for key %s", storeName, key));
+			}
+			final List<SongPlayCountBean> results = new ArrayList<>();
+			value.forEach(songPlayCount -> {
+				final ReadOnlyKeyValueStore<Long, Song> songStore =
+						queryableStoreRegistry.getQueryableStoreType(KafkaStreamsInteractiveQuerySample.ALL_SONGS, QueryableStoreTypes.<Long, Song>keyValueStore());
+
+				final Song song = songStore.get(songPlayCount.getSongId());
+				results.add(new SongPlayCountBean(song.getArtist(),song.getAlbum(), song.getName(),
+						songPlayCount.getPlays()));
+			});
+			return results;
+		}
+	}
+
+	/**
+	 * Serde for TopFiveSongs
+	 */
+	private static class TopFiveSerde implements Serde<TopFiveSongs> {
+
+		@Override
+		public void configure(final Map<String, ?> map, final boolean b) {
+
+		}
+
+		@Override
+		public void close() {
+
+		}
+
+		@Override
+		public Serializer<TopFiveSongs> serializer() {
+			return new Serializer<TopFiveSongs>() {
+				@Override
+				public void configure(final Map<String, ?> map, final boolean b) {
+				}
+
+				@Override
+				public byte[] serialize(final String s, final TopFiveSongs topFiveSongs) {
+					final ByteArrayOutputStream out = new ByteArrayOutputStream();
+					final DataOutputStream
+							dataOutputStream =
+							new DataOutputStream(out);
+					try {
+						for (SongPlayCount songPlayCount : topFiveSongs) {
+							dataOutputStream.writeLong(songPlayCount.getSongId());
+							dataOutputStream.writeLong(songPlayCount.getPlays());
+						}
+						dataOutputStream.flush();
+					} catch (IOException e) {
+						throw new RuntimeException(e);
+					}
+					return out.toByteArray();
+				}
+
+				@Override
+				public void close() {
+
+				}
+			};
+		}
+
+		@Override
+		public Deserializer<TopFiveSongs> deserializer() {
+			return new Deserializer<TopFiveSongs>() {
+				@Override
+				public void configure(final Map<String, ?> map, final boolean b) {
+
+				}
+
+				@Override
+				public TopFiveSongs deserialize(final String s, final byte[] bytes) {
+					if (bytes == null || bytes.length == 0) {
+						return null;
+					}
+					final TopFiveSongs result = new TopFiveSongs();
+
+					final DataInputStream
+							dataInputStream =
+							new DataInputStream(new ByteArrayInputStream(bytes));
+
+					try {
+						while(dataInputStream.available() > 0) {
+							result.add(new SongPlayCount(dataInputStream.readLong(),
+									dataInputStream.readLong()));
+						}
+					} catch (IOException e) {
+						throw new RuntimeException(e);
+					}
+					return result;
+				}
+
+				@Override
+				public void close() {
+
+				}
+			};
+		}
+	}
+
+	/**
+	 * Used in aggregations to keep track of the Top five songs
+	 */
+	static class TopFiveSongs implements Iterable<SongPlayCount> {
+		private final Map<Long, SongPlayCount> currentSongs = new HashMap<>();
+		private final TreeSet<SongPlayCount> topFive = new TreeSet<>((o1, o2) -> {
+			final int result = o2.getPlays().compareTo(o1.getPlays());
+			if (result != 0) {
+				return result;
+			}
+			return o1.getSongId().compareTo(o2.getSongId());
+		});
+
+		public void add(final SongPlayCount songPlayCount) {
+			if(currentSongs.containsKey(songPlayCount.getSongId())) {
+				topFive.remove(currentSongs.remove(songPlayCount.getSongId()));
+			}
+			topFive.add(songPlayCount);
+			currentSongs.put(songPlayCount.getSongId(), songPlayCount);
+			if (topFive.size() > 5) {
+				final SongPlayCount last = topFive.last();
+				currentSongs.remove(last.getSongId());
+				topFive.remove(last);
+			}
+		}
+
+		void remove(final SongPlayCount value) {
+			topFive.remove(value);
+			currentSongs.remove(value.getSongId());
+		}
+
+
+		@Override
+		public Iterator<SongPlayCount> iterator() {
+			return topFive.iterator();
+		}
+	}
+
+}

kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/Producers.java

@@ -0,0 +1,154 @@
+/*
+ * Copyright 2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.streams.interactive.query;
+
+import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig;
+import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerializer;
+import kafka.streams.interactive.query.avro.PlayEvent;
+import kafka.streams.interactive.query.avro.Song;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.LongSerializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.springframework.kafka.core.DefaultKafkaProducerFactory;
+import org.springframework.kafka.core.KafkaTemplate;
+
+import java.util.*;
+
+/**
+ * @author Soby Chacko
+ */
+public class Producers {
+
+	public static void main(String... args) throws Exception {
+
+		final Map<String, String> serdeConfig = Collections.singletonMap(
+				AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081");
+		final SpecificAvroSerializer<PlayEvent> playEventSerializer = new SpecificAvroSerializer<>();
+		playEventSerializer.configure(serdeConfig, false);
+		final SpecificAvroSerializer<Song> songSerializer = new SpecificAvroSerializer<>();
+		songSerializer.configure(serdeConfig, false);
+
+		final List<Song> songs = Arrays.asList(new Song(1L,
+						"Fresh Fruit For Rotting Vegetables",
+						"Dead Kennedys",
+						"Chemical Warfare",
+						"Punk"),
+				new Song(2L,
+						"We Are the League",
+						"Anti-Nowhere League",
+						"Animal",
+						"Punk"),
+				new Song(3L,
+						"Live In A Dive",
+						"Subhumans",
+						"All Gone Dead",
+						"Punk"),
+				new Song(4L,
+						"PSI",
+						"Wheres The Pope?",
+						"Fear Of God",
+						"Punk"),
+				new Song(5L,
+						"Totally Exploited",
+						"The Exploited",
+						"Punks Not Dead",
+						"Punk"),
+				new Song(6L,
+						"The Audacity Of Hype",
+						"Jello Biafra And The Guantanamo School Of "
+								+ "Medicine",
+						"Three Strikes",
+						"Punk"),
+				new Song(7L,
+						"Licensed to Ill",
+						"The Beastie Boys",
+						"Fight For Your Right",
+						"Hip Hop"),
+				new Song(8L,
+						"De La Soul Is Dead",
+						"De La Soul",
+						"Oodles Of O's",
+						"Hip Hop"),
+				new Song(9L,
+						"Straight Outta Compton",
+						"N.W.A",
+						"Gangsta Gangsta",
+						"Hip Hop"),
+				new Song(10L,
+						"Fear Of A Black Planet",
+						"Public Enemy",
+						"911 Is A Joke",
+						"Hip Hop"),
+				new Song(11L,
+						"Curtain Call - The Hits",
+						"Eminem",
+						"Fack",
+						"Hip Hop"),
+				new Song(12L,
+						"The Calling",
+						"Hilltop Hoods",
+						"The Calling",
+						"Hip Hop")
+
+		);
+
+		Map<String, Object> props = new HashMap<>();
+		props.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081");
+		props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
+		props.put(ProducerConfig.RETRIES_CONFIG, 0);
+		props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
+		props.put(ProducerConfig.LINGER_MS_CONFIG, 1);
+		props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432);
+		props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
+		props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, playEventSerializer.getClass());
+
+
+		Map<String, Object> props1 = new HashMap<>(props);
+		props1.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class);
+		props1.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, songSerializer.getClass());
+
+		DefaultKafkaProducerFactory<Long, Song> pf1 = new DefaultKafkaProducerFactory<>(props1);
+		KafkaTemplate<Long, Song> template1 = new KafkaTemplate<>(pf1, true);
+		template1.setDefaultTopic(KafkaStreamsInteractiveQuerySample.SONG_FEED);
+
+		songs.forEach(song -> {
+			System.out.println("Writing song information for '" + song.getName() + "' to input topic " +
+					KafkaStreamsInteractiveQuerySample.SONG_FEED);
+			template1.sendDefault(song.getId(), song);
+		});
+
+
+		DefaultKafkaProducerFactory<String, PlayEvent> pf = new DefaultKafkaProducerFactory<>(props);
+		KafkaTemplate<String, PlayEvent> template = new KafkaTemplate<>(pf, true);
+		template.setDefaultTopic(KafkaStreamsInteractiveQuerySample.PLAY_EVENTS);
+
+
+		final long duration = 60 * 1000L;
+		final Random random = new Random();
+
+		// send a play event every 100 milliseconds
+		while (true) {
+			final Song song = songs.get(random.nextInt(songs.size()));
+			System.out.println("Writing play event for song " + song.getName() + " to input topic " +
+					KafkaStreamsInteractiveQuerySample.PLAY_EVENTS);
+			template.sendDefault("uk", new PlayEvent(song.getId(), duration));
+
+			Thread.sleep(100L);
+		}
+	}
+
+}

kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/SongPlayCountBean.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright 2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.streams.interactive.query;
+
+import java.util.Objects;
+
+/**
+ * @author Soby Chacko
+ */
+public class SongPlayCountBean {
+
+	private String artist;
+	private String album;
+	private String name;
+	private Long plays;
+
+	public SongPlayCountBean() {}
+
+	public SongPlayCountBean(final String artist, final String album, final String name, final Long
+			plays) {
+
+		this.artist = artist;
+		this.album = album;
+		this.name = name;
+		this.plays = plays;
+	}
+
+	public String getArtist() {
+		return artist;
+	}
+
+	public void setArtist(final String artist) {
+		this.artist = artist;
+	}
+
+	public String getAlbum() {
+		return album;
+	}
+
+	public void setAlbum(final String album) {
+		this.album = album;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public Long getPlays() {
+		return plays;
+	}
+
+	public void setPlays(final Long plays) {
+		this.plays = plays;
+	}
+
+	@Override
+	public String toString() {
+		return "SongPlayCountBean{" +
+				"artist='" + artist + '\'' +
+				", album='" + album + '\'' +
+				", name='" + name + '\'' +
+				", plays=" + plays +
+				'}';
+	}
+
+	@Override
+	public boolean equals(final Object o) {
+		if (this == o) {
+			return true;
+		}
+		if (o == null || getClass() != o.getClass()) {
+			return false;
+		}
+		final SongPlayCountBean that = (SongPlayCountBean) o;
+		return Objects.equals(artist, that.artist) &&
+				Objects.equals(album, that.album) &&
+				Objects.equals(name, that.name) &&
+				Objects.equals(plays, that.plays);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(artist, album, name, plays);
+	}
+}

kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/application.yml

@@ -0,0 +1,25 @@
+spring.cloud.stream.bindings.input:
+  destination: play-events
+  consumer:
+    useNativeDecoding: true
+    headerMode: raw
+spring.cloud.stream.bindings.inputX:
+  destination: song-feed
+  consumer:
+    useNativeDecoding: true
+    headerMode: raw
+spring.cloud.stream.kafka.streams.bindings.input:
+  consumer:
+    keySerde: org.apache.kafka.common.serialization.Serdes$StringSerde
+    valueSerde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde
+spring.cloud.stream.kafka.streams.bindings.inputX:
+  consumer:
+    keySerde: org.apache.kafka.common.serialization.Serdes$LongSerde
+    valueSerde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde
+    materializedAs: all-songs
+spring.cloud.stream.kafka.streams.binder:
+  brokers: localhost #192.168.99.100
+  zkNodes: localhost #192.168.99.100
+  configuration:
+    schema.registry.url: http://localhost:8081
+    commit.interval.ms: 1000

kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/avro/kafka/streams/interactive/query/playevent.avsc

@@ -0,0 +1,8 @@
+{"namespace": "kafka.streams.interactive.query.avro",
+ "type": "record",
+ "name": "PlayEvent",
+ "fields": [
+     {"name": "song_id",   "type": "long"},
+     {"name": "duration", "type": "long"}
+ ]
+}

kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/avro/kafka/streams/interactive/query/song.avsc

@@ -0,0 +1,11 @@
+{"namespace": "kafka.streams.interactive.query.avro",
+ "type": "record",
+ "name": "Song",
+ "fields": [
+     {"name": "id",     "type": "long"},
+     {"name": "album",  "type": "string"},
+     {"name": "artist", "type": "string"},
+     {"name": "name",   "type": "string"},
+     {"name": "genre",  "type": "string"}
+ ]
+}

kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/avro/kafka/streams/interactive/query/songplaycount.avsc

@@ -0,0 +1,8 @@
+{"namespace": "kafka.streams.interactive.query.avro",
+ "type": "record",
+ "name": "SongPlayCount",
+ "fields": [
+     {"name": "song_id",  "type": "long"},
+     {"name": "plays",   "type": "long"}
+ ]
+}

kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/logback.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+    <appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%d{ISO8601} %5p %t %c{2}:%L - %m%n</pattern>
+        </encoder>
+    </appender>
+    <root level="INFO">
+        <appender-ref ref="stdout"/>
+    </root>
+    <logger name="org.apache.kafka.streams.processor.internals" level="WARN"/>
+</configuration>

kafka-streams-samples/kafka-streams-interactive-query-advanced/src/test/java/kafka/streams/interactive/query/KafkaStreamsInteractiveQueryTests.java

@@ -0,0 +1,18 @@
+package kafka.streams.interactive.query;
+
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest
+public class KafkaStreamsInteractiveQueryTests {
+
+	@Test
+	@Ignore
+	public void contextLoads() {
+	}
+
+}