diff --git a/kafka-batch-sample/src/main/java/com/example/demo/KafkaBatchSampleApplication.java b/kafka-batch-sample/src/main/java/com/example/demo/KafkaBatchSampleApplication.java index 490299b..4672757 100644 --- a/kafka-batch-sample/src/main/java/com/example/demo/KafkaBatchSampleApplication.java +++ b/kafka-batch-sample/src/main/java/com/example/demo/KafkaBatchSampleApplication.java @@ -100,7 +100,7 @@ class BatchProduce extends Base { @Bean Function, List>> consumer() { return list -> list.stream() - .map(string -> string.toUpperCase()) + .map(String::toUpperCase) .map(uppercasedString -> MessageBuilder.withPayload(uppercasedString).build()) .collect(Collectors.toList()); } diff --git a/kafka-streams-samples/kafka-streams-recoverable/.gitignore b/kafka-streams-samples/kafka-streams-recoverable/.gitignore new file mode 100644 index 0000000..2af7cef --- /dev/null +++ b/kafka-streams-samples/kafka-streams-recoverable/.gitignore @@ -0,0 +1,24 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +nbproject/private/ +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ \ No newline at end of file diff --git a/kafka-streams-samples/kafka-streams-recoverable/README.adoc b/kafka-streams-samples/kafka-streams-recoverable/README.adoc new file mode 100644 index 0000000..0085d47 --- /dev/null +++ b/kafka-streams-samples/kafka-streams-recoverable/README.adoc @@ -0,0 +1,26 @@ +== What is this app? + +This is an example of a Spring Cloud Stream processor using Kafka Streams support. + +This is a demonstration of utilizing the RecordRecoverableProcessor and DltAwareProcessor. + +The example increments a java.util.Suppler integer by 1 every 2 seconds, and publishes +to the `topic-in` kafka topic. + +There's then two processors, both of which are listening on `topic-in`. + +The first processor throws an exception in the workflow when the integer is divisible by 5. +The second processor throws an exception in the workflow when the integer is divisible by 4. + +This sample uses lambda expressions and thus requires Java 8+. + +=== Running the app: + +`docker-compose up -d` + +Go to the root of the repository and do: `./mvnw clean package` + +`java -jar target/kafka-streams-recoverable-0.0.1-SNAPSHOT.jar` + +The Supplier will automatically publish messages and the consumers +will write messages to the application log. diff --git a/kafka-streams-samples/kafka-streams-recoverable/docker-compose.yml b/kafka-streams-samples/kafka-streams-recoverable/docker-compose.yml new file mode 100644 index 0000000..3078c1d --- /dev/null +++ b/kafka-streams-samples/kafka-streams-recoverable/docker-compose.yml @@ -0,0 +1,19 @@ +version: '3' +services: + kafka: + image: wurstmeister/kafka + container_name: kafka-dlq + ports: + - "9092:9092" + environment: + - KAFKA_ADVERTISED_HOST_NAME=127.0.0.1 + - KAFKA_ADVERTISED_PORT=9092 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + depends_on: + - zookeeper + zookeeper: + image: wurstmeister/zookeeper + ports: + - "2181:2181" + environment: + - KAFKA_ADVERTISED_HOST_NAME=zookeeper diff --git a/kafka-streams-samples/kafka-streams-recoverable/mvnw b/kafka-streams-samples/kafka-streams-recoverable/mvnw new file mode 100755 index 0000000..2d1e3cf --- /dev/null +++ b/kafka-streams-samples/kafka-streams-recoverable/mvnw @@ -0,0 +1,225 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Migwn, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +echo $MAVEN_PROJECTBASEDIR +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/kafka-streams-samples/kafka-streams-recoverable/mvnw.cmd b/kafka-streams-samples/kafka-streams-recoverable/mvnw.cmd new file mode 100644 index 0000000..86846ae --- /dev/null +++ b/kafka-streams-samples/kafka-streams-recoverable/mvnw.cmd @@ -0,0 +1,143 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" + +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/kafka-streams-samples/kafka-streams-recoverable/pom.xml b/kafka-streams-samples/kafka-streams-recoverable/pom.xml new file mode 100644 index 0000000..a87bff5 --- /dev/null +++ b/kafka-streams-samples/kafka-streams-recoverable/pom.xml @@ -0,0 +1,144 @@ + + + 4.0.0 + + kafka-streams-recoverable + 0.0.1-SNAPSHOT + jar + + kafka-streams-recoverable + Demo project for Spring Boot + + + org.springframework.boot + spring-boot-starter-parent + 3.2.0 + + + + + 2023.0.0 + + + + + + org.springframework.cloud + spring-cloud-dependencies + ${spring-cloud.version} + pom + import + + + + + + + org.springframework.cloud + spring-cloud-stream + + + org.springframework.cloud + spring-cloud-stream-binder-kafka + + + org.springframework.cloud + spring-cloud-stream-binder-kafka-streams + + + org.springframework.kafka + spring-kafka + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.kafka + spring-kafka-test + test + + + org.apache.kafka + kafka-streams-test-utils + ${kafka.version} + test + + + + org.springframework.boot + spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-web + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + false + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + false + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + spring-releases + Spring Releases + https://repo.spring.io/libs-release-local + + false + + + + + diff --git a/kafka-streams-samples/kafka-streams-recoverable/src/main/java/kafka/streams/record/recoverable/KafkaStreamsRecoverableSample.java b/kafka-streams-samples/kafka-streams-recoverable/src/main/java/kafka/streams/record/recoverable/KafkaStreamsRecoverableSample.java new file mode 100644 index 0000000..bf2682d --- /dev/null +++ b/kafka-streams-samples/kafka-streams-recoverable/src/main/java/kafka/streams/record/recoverable/KafkaStreamsRecoverableSample.java @@ -0,0 +1,83 @@ +package kafka.streams.record.recoverable; + +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.processor.api.ProcessorSupplier; +import org.apache.kafka.streams.processor.api.Record; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cloud.stream.binder.kafka.streams.DltAwareProcessor; +import org.springframework.cloud.stream.binder.kafka.streams.DltPublishingContext; +import org.springframework.cloud.stream.binder.kafka.streams.RecordRecoverableProcessor; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.messaging.support.MessageBuilder; + +import java.time.Instant; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +import static java.lang.String.format; + +/** + * @author Steven Gantz + */ +@SpringBootApplication +public class KafkaStreamsRecoverableSample { + + int counter = 0; + + public static void main(String[] args) { + SpringApplication.run(KafkaStreamsRecoverableSample.class, args); + } + + // Create a new integer every 2 seconds + @Bean + @ConditionalOnProperty("${config.enable-supplier}") + public Supplier dataSupplier() { + return () -> { + var oldCounterValue = counter; + System.out.println(counter + " -> topic-in"); + counter += 1; + return String.valueOf(oldCounterValue); + }; + } + + // Using default recoverer using DltAwareProcessor, throw an exception on every number divisible by 5 + @Bean + public Function, KStream> defaultRecovererProcessor( + DltPublishingContext dltPublishingContext) { + return kstream -> kstream + .process(() -> new DltAwareProcessor<>(numRecord -> { + if (Integer.parseInt(numRecord.value()) % 5 == 0) { + // if the number is even, throw an exception + System.out.println("Throwing error with default recoverer"); + throw new RuntimeException("Something went wrong, Error"); + } else { + return new Record<>(numRecord.key(), "Success", Instant.now().getEpochSecond()); + } + }, "error-default-out", dltPublishingContext)); + } + + // Using custom recoverer using RecordRecoverableProcessor, throw an exception on every number divisible by 4 + @Bean + public Function, KStream> customRecovererProcessor( + DltPublishingContext dltPublishingContext) { + return kstream -> kstream + .process(() -> new RecordRecoverableProcessor<>(numRecord -> { + if (Integer.parseInt(numRecord.value()) % 4 == 0) { + throw new RuntimeException("Something went wrong, Error"); + } else { + return new Record<>(numRecord.key(), "Success", Instant.now().getEpochSecond()); + } + }, + (uuidIntegerRecord, e) -> { + System.out.println("Custom Error Logged"); + dltPublishingContext.getStreamBridge().send("error-custom-out", + format("%s:%s", uuidIntegerRecord, e.getMessage())); + } + )); + } +} diff --git a/kafka-streams-samples/kafka-streams-recoverable/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/kafka-streams-samples/kafka-streams-recoverable/src/main/resources/META-INF/additional-spring-configuration-metadata.json new file mode 100644 index 0000000..24c4a86 --- /dev/null +++ b/kafka-streams-samples/kafka-streams-recoverable/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -0,0 +1,8 @@ +{ + "properties": [ + { + "name": "config.enableSupplier", + "type": "java.lang.String", + "description": "Description for config.enableSupplier." + } +] } \ No newline at end of file diff --git a/kafka-streams-samples/kafka-streams-recoverable/src/main/resources/application.yml b/kafka-streams-samples/kafka-streams-recoverable/src/main/resources/application.yml new file mode 100644 index 0000000..654d8fb --- /dev/null +++ b/kafka-streams-samples/kafka-streams-recoverable/src/main/resources/application.yml @@ -0,0 +1,25 @@ +spring: + cloud: + function: + definition: dataSupplier;defaultRecovererProcessor;customRecovererProcessor + stream: + bindings: + dataSupplier-out-0: + destination: topic-in + + defaultRecovererProcessor-in-0: + destination: topic-in + group: defaultRecovererProcessor + defaultRecovererProcessor-out-0: + destination: default-out + + customRecovererProcessor-in-0: + destination: topic-in + group: customRecovererProcessor + customRecovererProcessor-out-0: + destination: custom-out + kafka: + binder: + brokers: localhost:9092 +config: + enableSupplier: true \ No newline at end of file diff --git a/kafka-streams-samples/kafka-streams-recoverable/src/main/resources/logback.xml b/kafka-streams-samples/kafka-streams-recoverable/src/main/resources/logback.xml new file mode 100644 index 0000000..870ac9e --- /dev/null +++ b/kafka-streams-samples/kafka-streams-recoverable/src/main/resources/logback.xml @@ -0,0 +1,12 @@ + + + + + %d{ISO8601} %5p %t %c{2}:%L - %m%n + + + + + + + \ No newline at end of file diff --git a/kafka-streams-samples/kafka-streams-recoverable/src/test/java/kafka/streams/record/recoverable/KafkaStreamsRecoverableSampleTest.java b/kafka-streams-samples/kafka-streams-recoverable/src/test/java/kafka/streams/record/recoverable/KafkaStreamsRecoverableSampleTest.java new file mode 100644 index 0000000..de1b53c --- /dev/null +++ b/kafka-streams-samples/kafka-streams-recoverable/src/test/java/kafka/streams/record/recoverable/KafkaStreamsRecoverableSampleTest.java @@ -0,0 +1,141 @@ +package kafka.streams.record.recoverable; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterEach; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.test.context.TestPropertySource; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static java.lang.Thread.sleep; +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest +@EmbeddedKafka( + partitions = 1, + controlledShutdown = true, + topics = {"topic-in", "default-out", "custom-out", "error-default-out", "error-custom-out"} +) +@TestPropertySource(properties = {"config.enableSupplier=false"}) +public class KafkaStreamsRecoverableSampleTest { + + @Autowired + EmbeddedKafkaBroker embeddedKafka; + + private static Consumer consumer; + + @AfterEach + public void tearDown() { + consumer.close(); + System.clearProperty("spring.cloud.stream.kafka.streams.binder.brokers"); + } + + @Test + public void testKafkaStreams() throws InterruptedException { + Map consumerProps = KafkaTestUtils.consumerProps("group", "false", embeddedKafka); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); + consumer = cf.createConsumer(); + embeddedKafka.consumeFromEmbeddedTopics(consumer, "topic-in", "default-out", "custom-out", + "error-default-out", "error-custom-out"); + System.setProperty("spring.cloud.stream.kafka.streams.binder.brokers", embeddedKafka.getBrokersAsString()); + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(senderProps); + try { + KafkaTemplate template = new KafkaTemplate<>(pf, true); + template.setDefaultTopic("topic-in"); + template.send(MessageBuilder.withPayload("1").build()); + template.send(MessageBuilder.withPayload("2").build()); + template.send(MessageBuilder.withPayload("3").build()); + template.send(MessageBuilder.withPayload("4").build()); + template.send(MessageBuilder.withPayload("5").build()); + + sleep(15000); + + ConsumerRecords consumerRecords = KafkaTestUtils.getRecords(consumer); + var topicIn = new ArrayList>(); + var defaultOut = new ArrayList>(); + var customOut = new ArrayList>(); + var errorDefaultOut = new ArrayList>(); + var errorCustomOut = new ArrayList>(); + + for(var record : consumerRecords.records("topic-in")) { + topicIn.add(record); + } + for(var record : consumerRecords.records("default-out")) { + defaultOut.add(record); + } + for(var record : consumerRecords.records("custom-out")) { + customOut.add(record); + } + for(var record : consumerRecords.records("error-default-out")) { + errorDefaultOut.add(record); + } + for(var record : consumerRecords.records("error-custom-out")) { + errorCustomOut.add(record); + } + + assertThat(topicIn.size()).isEqualTo(5); + assertThat(defaultOut.size()).isEqualTo(4); + assertThat(customOut.size()).isEqualTo(4); + assertThat(errorDefaultOut.size()).isEqualTo(1); + assertThat(errorCustomOut.size()).isEqualTo(1); + + validateTopicIn(topicIn); + validateDefaultOut(defaultOut); + validateCustomOut(customOut); + validateErrorDefaultOut(errorDefaultOut); + validateErrorCustomOut(errorCustomOut); + } + finally { + pf.destroy(); + } + } + + void validateTopicIn(List> records) { + assertThat(Integer.parseInt(records.get(0).value())).isEqualTo(1); + assertThat(Integer.parseInt(records.get(1).value())).isEqualTo(2); + assertThat(Integer.parseInt(records.get(2).value())).isEqualTo(3); + assertThat(Integer.parseInt(records.get(3).value())).isEqualTo(4); + assertThat(Integer.parseInt(records.get(4).value())).isEqualTo(5); + } + + void validateDefaultOut(List> records) { + assertThat(records.get(0).value()).isEqualTo("Success"); + assertThat(records.get(1).value()).isEqualTo("Success"); + assertThat(records.get(2).value()).isEqualTo("Success"); + assertThat(records.get(3).value()).isEqualTo("Success"); + } + + void validateCustomOut(List> records) { + assertThat(records.get(0).value()).isEqualTo("Success"); + assertThat(records.get(1).value()).isEqualTo("Success"); + assertThat(records.get(2).value()).isEqualTo("Success"); + assertThat(records.get(3).value()).isEqualTo("Success"); + } + + void validateErrorDefaultOut(List> records) { + assertThat(Integer.parseInt(records.get(0).value())).isEqualTo(5); + } + + void validateErrorCustomOut(List> records) { + Assertions.assertTrue(records.get(0).value().contains("value=4")); + Assertions.assertTrue(records.get(0).value().contains("Something went wrong, Error")); + } +} \ No newline at end of file diff --git a/kafka-streams-samples/pom.xml b/kafka-streams-samples/pom.xml index 69a4fed..4d29633 100644 --- a/kafka-streams-samples/pom.xml +++ b/kafka-streams-samples/pom.xml @@ -17,6 +17,7 @@ kafka-streams-global-table-join kafka-streams-message-channel kafka-streams-product-tracker + kafka-streams-recoverable kafka-streams-aggregate kafka-streams-to-rabbitmq-message-channel kafka-streams-inventory-count diff --git a/pom.xml b/pom.xml index c1ef422..d147b6f 100644 --- a/pom.xml +++ b/pom.xml @@ -34,7 +34,7 @@ partitioning-samples routing-samples spring-cloud-stream-schema-registry-integration - stream-function-batch-producer-consumer + batch-producer-consumer testing-samples transaction-kafka-samples