Spring/spring-integration-extensions
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INTEXT-44 - Add Kafka Support

Browse Source 
* Add Inbound adapter
* Add Outbound adapter
* Add serialization
* Initial Documentation
* Add unit tests
* Add async producer
* Support Kafka 0.8
* Add Kafka Sample

See: https://jira.springsource.org/browse/INTEXT-44
This commit is contained in:
Soby Chacko
2013-03-18 23:31:53 -04:00
committed by Gunnar Hillert
parent 0665dc9a86
commit e46cbfdcb4
92 changed files with 5735 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
samples/kafka/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
derby.log

5
samples/kafka/README.md Normal file
View File

@@ -0,0 +1,5 @@
Spring Integration Kafka Sample
================================
This sample demonstrates the usage of the *[Spring Integration Kafka][]* adapters.

46
samples/kafka/build.gradle Normal file
View File

@@ -0,0 +1,46 @@
description = 'Spring Integration Kafka Sample'
apply plugin: 'base'
apply plugin: 'java'
apply plugin: 'eclipse'
apply plugin: 'application'
apply plugin: 'idea'
apply plugin: 'maven'
repositories {
mavenLocal()
mavenCentral()
maven { url "http://repo.springsource.org/libs-snapshot" }
maven { url 'http://repo.springsource.org/plugins-release' }
}
dependencies {
compile "org.springframework:spring-beans:3.1.3.RELEASE"
compile "org.springframework:spring-context:3.1.3.RELEASE"
compile "org.springframework:spring-expression:3.1.3.RELEASE"
compile "org.springframework.integration:spring-integration-stream:2.2.0.RELEASE"
compile("org.springframework.integration:spring-integration-kafka:0.5.0.BUILD-SNAPSHOT") {
exclude module: 'log4j'
}
compile("log4j:log4j:1.2.15") {
exclude module: 'jms'
exclude module: 'jmx'
exclude module: 'jmxtools'
exclude module: 'jmxri'
}
compile "commons-logging:commons-logging:1.1.1"
}
task wrapper(type: Wrapper) {
description = 'Generates gradlew[.bat] scripts'
gradleVersion = '1.3'
}
eclipse {
project {
name = "spring-integration-kafka-sample"
}
}
defaultTasks 'clean', 'build'

7
samples/kafka/gradle.properties Normal file
View File

@@ -0,0 +1,7 @@
springVersion = 3.1.3.RELEASE
springIntegrationVersion = 2.2.0.RELEASE
springIntegrationKafkaVersion = 0.5.0.BUILD-SNAPSHOT
version = 0.5.0.BUILD-SNAPSHOT

BIN
samples/kafka/gradle/wrapper/gradle-wrapper.jar vendored Normal file
View File

Binary file not shown.

6
samples/kafka/gradle/wrapper/gradle-wrapper.properties vendored Normal file
View File

@@ -0,0 +1,6 @@
#Fri Sep 28 16:40:22 CST 2012
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=http\://services.gradle.org/distributions/gradle-1.1-bin.zip

164
samples/kafka/gradlew vendored Executable file
View File

@@ -0,0 +1,164 @@
#!/bin/bash
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS=""
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn ( ) {
echo "$*"
}
die ( ) {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MINGW* )
msys=true
;;
esac
# For Cygwin, ensure paths are in UNIX format before anything is touched.
if $cygwin ; then
[ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
fi
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/"
APP_HOME="`pwd -P`"
cd "$SAVED"
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query businessSystem maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin, switch paths to Windows format before running java
if $cygwin ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=$((i+1))
done
case $i in
(0) set -- ;;
(1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
function splitJvmOpts() {
JVM_OPTS=("$@")
}
eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"

90
samples/kafka/gradlew.bat vendored Normal file
View File

@@ -0,0 +1,90 @@
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS=
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto init
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto init
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:init
@rem Get command-line arguments, handling Windowz variants
if not "%OS%" == "Windows_NT" goto win9xME_args
if "%@eval[2+2]" == "4" goto 4NT_args
:win9xME_args
@rem Slurp the command line arguments.
set CMD_LINE_ARGS=
set _SKIP=2
:win9xME_args_slurp
if "x%~1" == "x" goto execute
set CMD_LINE_ARGS=%*
goto execute
:4NT_args
@rem Get arguments from the 4NT Shell from JP Software
set CMD_LINE_ARGS=%$
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega

27
samples/kafka/src/main/java/org/springframework/integration/samples/kafka/inbound/InboundRunner.java Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.samples.kafka.inbound;
import org.springframework.context.support.ClassPathXmlApplicationContext;
public class InboundRunner {
private static final String CONFIG = "kafkaInboundAdapterParserTests-context.xml";
public static void main(final String args[]) {
final ClassPathXmlApplicationContext ctx = new ClassPathXmlApplicationContext(CONFIG, InboundRunner.class);
ctx.start();
}
}

37
samples/kafka/src/main/java/org/springframework/integration/samples/kafka/outbound/CustomPartitioner.java Normal file
View File

@@ -0,0 +1,37 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.samples.kafka.outbound;
import kafka.producer.Partitioner;
/**
* @author Soby Chacko
*
* This class is for internal use only and therefore is at default access level
*/
class CustomPartitioner<T> implements Partitioner<T> {
/**
* Uses the key to calculate a partition bucket id for routing
* the data to the appropriate broker partition
* @return an integer between 0 and numPartitions-1
*/
@Override
public int partition(final T key, final int numPartitions) {
final String s = (String) key;
final Integer i = Integer.parseInt(s);
return i % numPartitions;
}
}

55
samples/kafka/src/main/java/org/springframework/integration/samples/kafka/outbound/OutboundRunner.java Normal file
View File

@@ -0,0 +1,55 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.samples.kafka.outbound;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.integration.MessageChannel;
import org.springframework.integration.support.MessageBuilder;
public class OutboundRunner {
private static final String CONFIG = "kafkaOutboundAdapterParserTests-context.xml";
private static final Log LOG = LogFactory.getLog(OutboundRunner.class);
public static void main(final String args[]) {
final ClassPathXmlApplicationContext ctx = new ClassPathXmlApplicationContext(CONFIG, OutboundRunner.class);
ctx.start();
final MessageChannel channel = ctx.getBean("inputToKafka", MessageChannel.class);
LOG.info(channel.getClass());
//sending 100,000 messages to Kafka server for topic test1
for (int i = 0; i < 500; i++) {
channel.send(
MessageBuilder.withPayload("hello Fom ob adapter test1 - " + i)
.setHeader("messageKey", String.valueOf(i))
.setHeader("topic", "test1").build());
LOG.info("message sent " + i);
}
//sending 5,000 messages to kafka server for topic test2
for (int i = 0; i < 50; i++) {
channel.send(
MessageBuilder.withPayload("hello Fom ob adapter test2 - " + i)
.setHeader("messageKey", String.valueOf(i))
.setHeader("topic", "test2").build());
LOG.info("message sent " + i);
}
}
}

56
samples/kafka/src/main/java/org/springframework/integration/samples/kafka/outbound/PartitionlessTransformer.java Normal file
View File

@@ -0,0 +1,56 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.samples.kafka.outbound;
import org.springframework.integration.Message;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.integration.transformer.Transformer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author Soby Chacko
*/
public class PartitionlessTransformer implements Transformer {
@Override
@SuppressWarnings("unchecked")
public Message<?> transform(final Message<?> message) {
final Map<String, Map<Integer, List<Object>>> origData =
(Map<String, Map<Integer, List<Object>>>) message.getPayload();
final Map<String, List<Object>> nonPartitionedData = new HashMap<String, List<Object>>();
for(final String topic : origData.keySet()) {
final Map<Integer, List<Object>> partitionedData = origData.get(topic);
final Collection<List<Object>> nonPartitionedDataFromTopic = partitionedData.values();
final List<Object> mergedList = new ArrayList<Object>();
for (final List<Object> l : nonPartitionedDataFromTopic){
mergedList.addAll(l);
}
nonPartitionedData.put(topic, mergedList);
}
return MessageBuilder.withPayload(nonPartitionedData).build();
}
}

10
samples/kafka/src/main/resources/log4j.properties Normal file
View File

@@ -0,0 +1,10 @@
log4j.rootCategory=WARN, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{HH:mm:ss.SSS} %-5p [%t][%c] %m%n
#log4j.category.org.springframework.integration=WARN
log4j.category.org.springframework.integration.kafka=INFO
log4j.category.org.springframework.integration.samples.kafka=INFO
log4j.category.kafka.consumer=ERROR

56
samples/kafka/src/main/resources/org/springframework/integration/samples/kafka/inbound/kafkaInboundAdapterParserTests-context.xml Normal file
View File

@@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:stream="http://www.springframework.org/schema/integration/stream"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xsi:schemaLocation="http://www.springframework.org/schema/integration/stream http://www.springframework.org/schema/integration/stream/spring-integration-stream.xsd
http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<int:channel id="inputFromKafka"/>
<int:channel id="inputFromTransformer"/>
<stream:stdout-channel-adapter id="stdout" channel="inputFromKafka" append-newline="true"/>
<int-kafka:zookeeper-connect id="zookeeperConnect" zk-connect="localhost:2181"
zk-connection-timeout="6000"
zk-session-timeout="6000"
zk-sync-time="2000"/>
<int-kafka:inbound-channel-adapter id="kafkaInboundChannelAdapter"
kafka-consumer-context-ref="consumerContext"
auto-startup="false"
channel="inputFromKafka">
<int:poller fixed-delay="1" time-unit="MILLISECONDS"/>
</int-kafka:inbound-channel-adapter>
<bean id="kafkaDecoder" class="org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaDecoder">
<constructor-arg type="java.lang.Class" value="java.lang.String"/>
</bean>
<int-kafka:consumer-context id="consumerContext"
consumer-timeout="1000"
zookeeper-connect="zookeeperConnect">
<int-kafka:consumer-configurations>
<int-kafka:consumer-configuration group-id="default"
value-decoder="kafkaDecoder"
key-decoder="kafkaDecoder"
max-messages="5000">
<int-kafka:topic id="test1" streams="4"/>
</int-kafka:consumer-configuration>
<int-kafka:consumer-configuration group-id="default1"
max-messages="5">
<int-kafka:topic id="test2" streams="4"/>
</int-kafka:consumer-configuration>
</int-kafka:consumer-configurations>
</int-kafka:consumer-context>
<bean id="partitionlessTransformer" class="org.springframework.integration.samples.kafka.outbound.PartitionlessTransformer"/>
<int:transformer ref="partitionlessTransformer" method="transform"
input-channel="inputFromKafka"
output-channel="inputFromTransformer"/>
</beans>

45
samples/kafka/src/main/resources/org/springframework/integration/samples/kafka/outbound/kafkaOutboundAdapterParserTests-context.xml Normal file
View File

@@ -0,0 +1,45 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xmlns:task="http://www.springframework.org/schema/task"
xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task.xsd">
<int:channel id="inputToKafka">
<int:queue/>
</int:channel>
<int-kafka:outbound-channel-adapter kafka-producer-context-ref="kafkaProducerContext"
auto-startup="false"
channel="inputToKafka">
<int:poller fixed-delay="1000" time-unit="MILLISECONDS" receive-timeout="0" task-executor="taskExecutor"/>
</int-kafka:outbound-channel-adapter>
<task:executor id="taskExecutor" pool-size="5" keep-alive="120" queue-capacity="500"/>
<bean id="kafkaEncoder" class="org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaEncoder">
<constructor-arg value="java.lang.String" />
</bean>
<bean id="customPartitioner" class="org.springframework.integration.samples.kafka.outbound.CustomPartitioner"/>
<int-kafka:producer-context id="kafkaProducerContext">
<int-kafka:producer-configurations>
<int-kafka:producer-configuration broker-list="localhost:9092"
key-class-type="java.lang.String"
value-class-type="java.lang.String"
topic="test1"
value-encoder="kafkaEncoder"
key-encoder="kafkaEncoder"
compression-codec="default"
partitioner="customPartitioner"/>
<int-kafka:producer-configuration broker-list="localhost:9092"
topic="test2"
compression-codec="default"/>
</int-kafka:producer-configurations>
</int-kafka:producer-context>
</beans>

8
spring-integration-kafka/.gitignore vendored Normal file
View File

@@ -0,0 +1,8 @@
.classpath
.project
.settings/
.gradle
build
target/
bin/
derby.log

294
spring-integration-kafka/README.md Normal file
View File

@@ -0,0 +1,294 @@
Spring Integration Kafka Adapter
=================================================
Welcome to the *Spring Integration Kafka adapter*. Apache Kafka is a distributed publish-subscribe messaging system that is designed for handling terra bytes of high throughput
data at constant time. For more information on Kafka and its design goals, please see [Kafka main page](http://kafka.apache.org/)
Spring Integration Kafka adapters are built for Kafka 0.8 and since 0.8 is not backward compatible with any previous versions, Spring Integration will not
support any Kafka versions prior to 0.8. As of this writing, Kafka 0.8 is still WIP.
Checking out and building
-----------------------------
Currently Spring Integration Kafka adapter is built against kafka 0.8 that is backed by
Scala 2.9.2.
In order to build the project:
./gradlew build
In order to install this into your local maven cache:
./gradlew install
Spring Integration Kafka project currently supports the following two components. Please keep in mind that
this is very early stage in development and do not yet fully make use of all the features that Kafka provides.
* Outbound Channel Adapter
* Inbound Channel Adapter based on the High level consumer API
Outbound Channel Adapter:
--------------------------------------------
The Outbound channel adapter is used to send messages to Kafka. Messages are read from a Spring Integration channel. One can specify this channel in the application context and then wire
this in the application where messages are sent to kafka.
Once a channel is configured, then messages can be sent to Kafka through this channel. Obviously, Spring Integration specific messages are sent to the adapter and then it will
internally get converted into Kafka messages before sending. In the current version of the outbound adapter,
you have to specify a message key and the topic as header values and the message to send as the payload.
Here is an example.
```java
final MessageChannel channel = ctx.getBean("inputToKafka", MessageChannel.class);
channel.send(
MessageBuilder.withPayload(payload).
setHeader("messageKey", "key")
.setHeader("topic", "test").build());
```
This would create a message with a payload. In addition to this, it also creates two header entries as key/value pairs - one for
the message key and another for the topic that this message belongs to.
Here is how kafka outbound channel adapter is configured:
```xml
<int-kafka:outbound-channel-adapter id="kafkaOutboundChannelAdapter"
kafka-producer-context-ref="kafkaProducerContext"
auto-startup="false"
channel="inputToKafka">
<int:poller fixed-delay="1000" time-unit="MILLISECONDS" receive-timeout="0" task-executor="taskExecutor"/>
</int-kafka:outbound-channel-adapter>
```
The key aspect in this configuration is the producer-context-ref. Producer context contains all the producer configuration for all the topics that this adapter is expected to handle.
A channel in which messages are arriving is configured with the adapter and therefore
any message sent to that channel will be handled by this adapter. You can also configure a poller
depending on the
type of the channel used. For example, in the above configuration, we use a queue based channel
and thus a poller is configured with a task executor. If no messages are available in the queue it will timeout immediately because of
the receive-timeout configuration. Then it will poll again with a delay of 1 second.
Producer context is at the heart of the kafka outbound adapter. Here is an example of how it is configured.
```xml
<int-kafka:producer-context id="kafkaProducerContext">
<int-kafka:producer-configurations>
<int-kafka:producer-configuration broker-list="localhost:9092"
key-class-type="java.lang.String"
value-class-type="java.lang.String"
topic="test1"
value-encoder="kafkaEncoder"
key-encoder="kafkaEncoder"
compression-codec="default"/>
<int-kafka:producer-configuration broker-list="localhost:9092"
topic="test2"
compression-codec="default"
async="true"/>
</int-kafka:producer-configurations>
</int-kafka:producer-context>
```
There are a few things going on here. So, lets go one by one. First of all, producer context is simply a holder of, as the name
indicates, a context for the Kafa producer. It contains one ore more producer configurations. Each producer configuration
is ultimately gets translated into a Kafka native producer. Each producer configuration is per topic based right now.
If you go by the above example, there are two producers generated from this configuration - one for topic named
test1 and another for test2. Each producer can take the following:
broker-list list of comma separated brokers that this producer connects to
topic topic name
compression-codec any compression to be used. Default is no compression. Supported compression codec are gzip and snappy. Anything else would
result in no compression
value-encoder serializer to be used for encoding messages.
key-encoder serializer to be used for encoding the partition key
key-class-type Type of the key class. This will be ignored if no key-encoder is provided
value-class-type The type of the value class. This will be ignored if no value-encoder is provided.
partitioner custom implementation of a Kafka Partitioner interface.
async true/false - default is false. Setting this to true would make the Kafka producer to use
an async producer
batch-num-messages number of messages to batch at the producer. If async is false, then this has no effect.
The value-encoder and key-encoder are referring to other spring beans. They are essentially implementations of an
interface provided by Kafka, the Encoder interface. Similarly, partitioner also refers a Spring bean which implements
the Kafka Partitioner interface.
Here is an example of configuring an encoder.
```xml
<bean id="kafkaEncoder" class="org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaEncoder">
<constructor-arg value="java.lang.String" />
</bean>
```
Spring Integration Kafaka adapter provides Apache Avro backed encoders out of the box, as this is a popular choice
for serialization in the big data spectrum. If no encoders are specified as beans, the default encoders provided
by Kafka will be used. On that not, if the encoder is configured only for the message and not for the key, the same encoder
will be used for both. These are standard Kafka behaviors. Spring Integration Kafka adapter does simply enforce those behaviours.
Kafka default encoder expects the data to come as byte arrays and it is a no-op encoder, i.e. it just takes the byte array as it is.
When default encoders are used, there are two ways a message can be sent.
Either, the sender of the message to the channel
can simply put byte arrays as message key and payload. Or, the key and value can be sent as Java Serializable object.
In the latter case, the Kafka adapter will automatically convert them to byte arrays before sending to Kafka broker.
If the encoders are default and the objets sent are not serializalbe, then that would cause an error. By providing explicit encoders
it is totally up to the developer to configure how the objects are serialized. In that case, the objects may or may not implement
the Serializable interface.
Encoding String for key and value is a very common use case and Kafka provides a StringEncoder out of the box. It takes a Kafka specific VerifiableProperties object
along with its
constructor that wraps a regular Java.util.Properties object. The StringEncoder is great when writing a
direct Java client that talks to Kafka.
However, when using Spring Integration Kafka adapter, it introduces unnecessary steps to create these
properties objects. Therefore, we provide a wrapper class for this same StringEncoder as part of the SI kafka support, which makes
using it from Spring a bit easier. You can inject
any properties to it in the Spring way. Kafka StringEncoder looks at a specific property for the type of encoding scheme used.
In the wrapper bean provided, this property can simply be injected as a value without constructing any other objects. Spring Integration provided StringEncoder is available
in the package org.springframework.integration.kafka.serializer.common.StringEncoder. The avro support for serialization is
also available in a package called avro under serializer.
Inbound Channel Adapter:
--------------------------------------------
The Inbound channel adapter is used to consume messages from Kafka. These messages will be placed into a channel as Spring Integration specific Messages.
Kafka provides two types of consumer API's primarily. One is called the high level consumer and the other is the Simple Consumer. Highlevel consumer is
pretty complex inside. Nonetheless, for the client, using the high level API is straightforward. Although easy to use, High level consumer
does not provide any offset management. So, if you want to rewind and re-fetch messages, it is not possible to do so using the
high level consumer API. Offsets are managed by the Zookeeper internally in the high level consumer. If your use case does not require any offset management
or re-reading messages from the same consumer, then high level consumer is a perfect fit. Spring Integration Kafka inbound channel adapter
currently supports only the high level consumer. Here are the details of configuring one.
```xml
<int-kafka:inbound-channel-adapter id="kafkaInboundChannelAdapter"
kafka-consumer-context-ref="consumerContext"
auto-startup="false"
channel="inputFromKafka">
<int:poller fixed-delay="10" time-unit="MILLISECONDS" max-messages-per-poll="5"/>
</int-kafka:inbound-channel-adapter>
```
Since this inbound channel adapter uses a Polling Channel under the hood, it must be configured with a Poller. A notable difference
between the poller configured with this inbound adapter and other pollers is that the receive-timeout specified here
does not have any effect. The reason for this is because of the way Kafka implements iterators on the consumer stream.
It is using a BlockingQueue internally and thus it would wait indefinitely. Instead of interrupting the underlying thread,
we are leveraging on direct Kafka support for consumer time out. It is configured on the consumer context. Everything else
is pretty much the same as in a regular inbound adapter. Any messages that it receives will be sent to the channel configured with it.
Inbound Kafka Adapter must specify a kafka-consumer-context-ref element and here is how it is configured:
```xml
<int-kafka:consumer-context id="consumerContext"
consumer-timeout="4000"
zookeeper-connect="zookeeperConnect">
<int-kafka:consumer-configurations>
<int-kafka:consumer-configuration group-id="default"
value-decoder="valueDecoder"
key-decoder="valueDecoder"
max-messages="5000">
<int-kafka:topic id="test1" streams="4"/>
<int-kafka:topic id="test2" streams="4"/>
</int-kafka:consumer-configuration>
</int-kafka:consumer-configurations>
</int-kafka:consumer-context>
```
Consumer context requires a reference to a zookeeper-connect which dictates all the zookeeper specific configuration details.
Here is how a zookeeper-connect is configured.
```xml
<int-kafka:zookeeper-connect id="zookeeperConnect" zk-connect="localhost:2181" zk-connection-timeout="6000"
zk-session-timeout="6000"
zk-sync-time="2000" />
```
zk-connect attribute is where you would specify the zookeeper connection. All the other attributes get translated into their
zookeeper counter-part attributes by the consumer.
In the above consumer context, you can also specify a consumer-timeout value which would be used to
timeout the consumer in case of no messages to consume.
This timeout would be applicable to all the streams (threads) in the consumer.
The default value for this in Kafka is -1 which would make it wait
indefinitely. However, Sping Integration overrides it to be 5 seconds in order to make sure that no
threads are blocking indefinitely in the lifecycle of the application and thereby
giving them a chance to free up any resources or locks that they hold. It is recommended to
override this value so as to meet any specific use case requirements.
By providing a reasonable consumer-timeout and a fixed-delay value on the poller,
this inbound adapter is capable of simulating a message driven behaviour.
consumer context takes consumer-configurations which are at the center piece of the inbound adapter. It is a group of one or more
consumer-configuration elements which consists of a consumer group dictated by the group-id. Each consumer-configuration
can be configured with one or more kafka-topic.
In the above example provided, we have a single consumer-configuration that consumes messages from two topics each having 4 streams.
These streams are fundamentally same as the number of partitions that a topic is configured
with in the producer. For instance, if you configure your topic with
4 partitions, then the maximum number of streams that you may have in the consumer is also 4.
Any more than this would be a no-op.
If you have less number of streams than the available partitions, then messages from
multiple partitions will be sent to available streams.
Therefore, it is a good practice to limit the number of streams for a topic in the consumer
configuration to the number of partitions configured for the topic. There may be situations
in which a partition may be gone during runtime and in that case the stream receiving
data from the partition will simply timeout and whenever this partition comes back,
it would start read data from it again.
Consumer configuration can also be configured with optional decoders for key and value.
The default ones provided by Kafka are basically no-ops and would consume as byte arrays.
If you provide an encoder for key/value in the producer, then it is recommended to provide
corresponding decoders.
Spring Integration Kafka adapter gives Apache Avro based data serialization components
out of the box. You can use any serialization component for this purpose.
Here is how you would configure a kafka decoder bean that is Avro backed.
```xml
<bean id="kafkaDecoder" class="org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaDecoder">
<constructor-arg type="java.lang.Class" value="java.lang.String" />
</bean>
```
Another important attribute for the consumer-configuration is the max-messages.
Please note that this is different from the max-messages-per-poll configured on the inbound adapter
element.
There it means the number of times the receive method called on the adapter.
The max-messages on consumer configuration is different. Kafka is used mainly for big data purposes
and usually that means the influx of large amount of data constantly. Because of this,
each time a receive is invoked on the adapter, you would basically get a collection of messages.
The maximum number of messages to retrieve for a topic in each execution of the
receive is what configured through the max-messages attribute on the consumer-configuration.
Basically, if the use case is to receive a constant stream of
large number of data, simply specifying a receive-timeout alone would not be enough.
You would also need to specify the max number of messages to receive.
The type of the payload of the Message returned by the adapter is the following:
```java
Map<String, Map<Integer, List<Object>>>
```
It is a java.util.Map that contains the topic string consumed as the key and another Map as the value.
The inner map's key is the stream (partition) number and value is a list of message payloads.
The reason for this complex return type is
due to the way Kafka orders the messages. In the high level consumer,
all the messages received in a single stream for a single partition
are guaranteed to be in order. For example, if I have a topic named test configured with
4 partitions and I have 4 corresponding streams
in the consumer, then I would receive data in all the consumer streams in the same order
as they were put in the corresponding partitions. This is another reason to set the number of
consumer streams for a topic same
as the number of broker partitions configured for that topic. Lets say that the number of streams
are less than the number of partitions. Then, normally, there is no
guarantee for any order other than just the fact that a single stream will contain messages
from multiple partitions and the messages from a given single partition received will
still be kept contiguously. By that time probably there is no way to find out which set of messages came from which partition.
By providing this complex map that contains the partition information for the topic, we make sure that the order sent by the producer
is preserved even if the number of streams used was less than the number of broker partitions.
A downstream component which receives the data from the inbound adapter can cast the SI payload to the above
Map.
If your use case does not require ordering of messages during consumption, then you can easily pass this
payload to a standard SI transformer and just get a full dump of the actual payload sent by Kafka.

304
spring-integration-kafka/build.gradle Normal file
View File

@@ -0,0 +1,304 @@
description = 'Spring Integration Kafka Adapter'
buildscript {
repositories {
maven { url 'https://repo.springsource.org/plugins-snapshot' }
}
dependencies {
classpath 'org.springframework.build.gradle:docbook-reference-plugin:0.1.5'
}
}
apply plugin: 'java'
apply from: "${rootProject.projectDir}/publish-maven.gradle"
apply plugin: 'eclipse'
apply plugin: 'idea'
group = 'org.springframework.integration'
repositories {
mavenLocal()
mavenCentral()
maven { url 'http://repo.springsource.org/libs-milestone' }
maven { url 'http://repo.springsource.org/plugins-release' }
maven { url 'https://repo.springsource.org/libs-snapshot' }
}
sourceCompatibility=1.6
targetCompatibility=1.6
ext {
junitVersion = '4.10'
log4jVersion = '1.2.12'
mockitoVersion = '1.9.0'
springVersion = '3.1.3.RELEASE'
springIntegrationVersion = '2.2.0.RELEASE'
idPrefix = 'kafka'
}
eclipse {
project {
natures += 'org.springframework.ide.eclipse.core.springnature'
}
}
sourceSets {
test {
resources {
srcDirs = ['src/test/resources', 'src/test/java']
}
}
}
// See http://www.gradle.org/docs/current/userguide/dependency_management.html#sub:configurations
// and http://www.gradle.org/docs/current/dsl/org.gradle.api.artifacts.ConfigurationContainer.html
configurations {
jacoco //Configuration Group used by Sonar to provide Code Coverage using JaCoCo
}
dependencies {
compile "org.springframework:spring-beans:$springVersion"
compile "org.springframework:spring-context:$springVersion"
compile "org.springframework:spring-expression:$springVersion"
compile "org.springframework.integration:spring-integration-core:$springIntegrationVersion"
compile("org.apache:kafka_2.9.2:0.8.0-SNAPSHOT") {
exclude module: 'log4j'
}
compile("org.apache.zookeeper:zookeeper:3.3.4") {
exclude module: 'log4j'
exclude module: 'jline'
}
compile "org.scala-lang:scala-library:2.9.2"
compile "org.scala-lang:scala-compiler:2.9.2"
compile("log4j:log4j:1.2.15") {
exclude module: 'jms'
exclude module: 'jmx'
exclude module: 'jmxtools'
exclude module: 'jmxri'
}
compile "org.slf4j:slf4j-simple:1.6.4"
compile "net.sf.jopt-simple:jopt-simple:3.2"
compile "org.xerial.snappy:snappy-java:1.0.4.1"
//runtime "zkclient:zkclient:20120522"
runtime "com.101tec:zkclient:0.2"
runtime "com.yammer.metrics:metrics-core:2.2.0"
runtime "com.yammer.metrics:metrics-annotation:2.2.0"
compile "org.apache.avro:avro:1.7.3"
compile "org.apache.avro:avro-compiler:1.7.3"
testCompile "org.springframework.integration:spring-integration-test:$springIntegrationVersion"
testCompile "junit:junit-dep:$junitVersion"
testCompile("log4j:log4j:$log4jVersion") {
exclude module: 'jms'
exclude module: 'jmx'
exclude module: 'jmxtools'
exclude module: 'jmxri'
}
testCompile "org.mockito:mockito-all:$mockitoVersion"
testCompile "org.springframework:spring-test:$springVersion"
testCompile "org.springframework.integration:spring-integration-stream:$springIntegrationVersion"
jacoco group: "org.jacoco", name: "org.jacoco.agent", version: "0.5.6.201201232323", classifier: "runtime"
}
// enable all compiler warnings; individual projects may customize further
ext.xLintArg = '-Xlint:all'
[compileJava, compileTestJava]*.options*.compilerArgs = [xLintArg]
test {
// suppress all console output during testing unless running `gradle -i`
logging.captureStandardOutput(LogLevel.INFO)
jvmArgs "-javaagent:${configurations.jacoco.asPath}=destfile=${buildDir}/jacoco.exec,includes=*"
}
task sourcesJar(type: Jar) {
classifier = 'sources'
from sourceSets.main.allJava
}
task javadocJar(type: Jar) {
classifier = 'javadoc'
from javadoc
}
artifacts {
archives sourcesJar
archives javadocJar
}
apply plugin: 'docbook-reference'
reference {
sourceDir = file('src/reference/docbook')
}
apply plugin: 'sonar'
sonar {
if (rootProject.hasProperty('sonarHostUrl')) {
server.url = rootProject.sonarHostUrl
}
database {
if (rootProject.hasProperty('sonarJdbcUrl')) {
url = rootProject.sonarJdbcUrl
}
if (rootProject.hasProperty('sonarJdbcDriver')) {
driverClassName = rootProject.sonarJdbcDriver
}
if (rootProject.hasProperty('sonarJdbcUsername')) {
username = rootProject.sonarJdbcUsername
}
if (rootProject.hasProperty('sonarJdbcPassword')) {
password = rootProject.sonarJdbcPassword
}
}
project {
dynamicAnalysis = "reuseReports"
withProjectProperties { props ->
props["sonar.core.codeCoveragePlugin"] = "jacoco"
props["sonar.jacoco.reportPath"] = "${buildDir.name}/jacoco.exec"
}
}
logger.info("Sonar parameters used: server.url='${server.url}'; database.url='${database.url}'; database.driverClassName='${database.driverClassName}'; database.username='${database.username}'")
}
task api(type: Javadoc) {
group = 'Documentation'
description = 'Generates the Javadoc API documentation.'
title = "${rootProject.description} ${version} API"
options.memberLevel = org.gradle.external.javadoc.JavadocMemberLevel.PROTECTED
options.author = true
options.header = rootProject.description
options.overview = 'src/api/overview.html'
source = sourceSets.main.allJava
classpath = project.sourceSets.main.compileClasspath
destinationDir = new File(buildDir, "api")
}
task schemaZip(type: Zip) {
group = 'Distribution'
classifier = 'schema'
description = "Builds -${classifier} archive containing all " +
"XSDs for deployment at static.springframework.org/schema."
def Properties schemas = new Properties();
def shortName = idPrefix.replaceFirst("${idPrefix}-", '')
project.sourceSets.main.resources.find {
it.path.endsWith('META-INF/spring.schemas')
}?.withInputStream { schemas.load(it) }
for (def key : schemas.keySet()) {
File xsdFile = project.sourceSets.main.resources.find {
it.path.endsWith(schemas.get(key))
}
assert xsdFile != null
into ("integration/${shortName}") {
from xsdFile.path
}
}
}
task docsZip(type: Zip) {
group = 'Distribution'
classifier = 'docs'
description = "Builds -${classifier} archive containing api and reference " +
"for deployment at static.springframework.org/spring-integration/docs."
from('src/dist') {
include 'changelog.txt'
}
from (api) {
into 'api'
}
from (reference) {
into 'reference'
}
}
task distZip(type: Zip, dependsOn: [docsZip, schemaZip]) {
group = 'Distribution'
classifier = 'dist'
description = "Builds -${classifier} archive, containing all jars and docs, " +
"suitable for community download page."
ext.baseDir = "${project.name}-${project.version}";
from('src/dist') {
include 'readme.txt'
include 'license.txt'
include 'notice.txt'
into "${baseDir}"
}
from(zipTree(docsZip.archivePath)) {
into "${baseDir}/docs"
}
from(zipTree(schemaZip.archivePath)) {
into "${baseDir}/schema"
}
into ("${baseDir}/libs") {
from project.jar
from project.sourcesJar
from project.javadocJar
}
}
// Create an optional "with dependencies" distribution.
// Not published by default; only for use when building from source.
task depsZip(type: Zip, dependsOn: distZip) { zipTask ->
group = 'Distribution'
classifier = 'dist-with-deps'
description = "Builds -${classifier} archive, containing everything " +
"in the -${distZip.classifier} archive plus all dependencies."
from zipTree(distZip.archivePath)
gradle.taskGraph.whenReady { taskGraph ->
if (taskGraph.hasTask(":${zipTask.name}")) {
def projectName = rootProject.name
def artifacts = new HashSet()
rootProject.configurations.runtime.resolvedConfiguration.resolvedArtifacts.each { artifact ->
def dependency = artifact.moduleVersion.id
if (!projectName.equals(dependency.name)) {
artifacts << artifact.file
}
}
zipTask.from(artifacts) {
into "${distZip.baseDir}/deps"
}
}
}
}
artifacts {
archives distZip
archives docsZip
archives schemaZip
}
task dist(dependsOn: assemble) {
group = 'Distribution'
description = 'Builds -dist, -docs and -schema distribution archives.'
}
task wrapper(type: Wrapper) {
description = 'Generates gradlew[.bat] scripts'
gradleVersion = '1.3'
}

1
spring-integration-kafka/gradle.properties Normal file
View File

@@ -0,0 +1 @@
version=0.5.0.BUILD-SNAPSHOT

BIN
spring-integration-kafka/gradle/wrapper/gradle-wrapper.jar vendored Normal file
View File

Binary file not shown.

6
spring-integration-kafka/gradle/wrapper/gradle-wrapper.properties vendored Normal file
View File

@@ -0,0 +1,6 @@
#Wed Sep 05 13:34:36 EDT 2012
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=http\://services.gradle.org/distributions/gradle-1.1-bin.zip

164
spring-integration-kafka/gradlew vendored Executable file
View File

@@ -0,0 +1,164 @@
#!/bin/bash
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS=""
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn ( ) {
echo "$*"
}
die ( ) {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MINGW* )
msys=true
;;
esac
# For Cygwin, ensure paths are in UNIX format before anything is touched.
if $cygwin ; then
[ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
fi
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/"
APP_HOME="`pwd -P`"
cd "$SAVED"
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query businessSystem maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin, switch paths to Windows format before running java
if $cygwin ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=$((i+1))
done
case $i in
(0) set -- ;;
(1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
function splitJvmOpts() {
JVM_OPTS=("$@")
}
eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"

90
spring-integration-kafka/gradlew.bat vendored Normal file
View File

@@ -0,0 +1,90 @@
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS=
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto init
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto init
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:init
@rem Get command-line arguments, handling Windowz variants
if not "%OS%" == "Windows_NT" goto win9xME_args
if "%@eval[2+2]" == "4" goto 4NT_args
:win9xME_args
@rem Slurp the command line arguments.
set CMD_LINE_ARGS=
set _SKIP=2
:win9xME_args_slurp
if "x%~1" == "x" goto execute
set CMD_LINE_ARGS=%*
goto execute
:4NT_args
@rem Get arguments from the 4NT Shell from JP Software
set CMD_LINE_ARGS=%$
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega

61
spring-integration-kafka/publish-maven.gradle Normal file
View File

@@ -0,0 +1,61 @@
apply plugin: 'maven'
ext.optionalDeps = []
ext.providedDeps = []
ext.optional = { optionalDeps << it }
ext.provided = { providedDeps << it }
install {
repositories.mavenInstaller {
customizePom(pom, project)
}
}
def customizePom(pom, gradleProject) {
pom.whenConfigured { generatedPom ->
// respect 'optional' and 'provided' dependencies
gradleProject.optionalDeps.each { dep ->
generatedPom.dependencies.find { it.artifactId == dep.name }?.optional = true
}
gradleProject.providedDeps.each { dep ->
generatedPom.dependencies.find { it.artifactId == dep.name }?.scope = 'provided'
}
// eliminate test-scoped dependencies (no need in maven central poms)
generatedPom.dependencies.removeAll { dep ->
dep.scope == 'test'
}
// add all items necessary for maven central publication
generatedPom.project {
name = gradleProject.description
description = gradleProject.description
url = 'https://github.com/SpringSource/spring-integration-extensions'
organization {
name = 'SpringSource'
url = 'http://springsource.org'
}
licenses {
license {
name 'The Apache Software License, Version 2.0'
url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
distribution 'repo'
}
}
scm {
url = 'https://github.com/SpringSource/spring-integration-extensions'
connection = 'scm:git:git://github.com/SpringSource/spring-integration-extensions'
developerConnection = 'scm:git:git://github.com/SpringSource/spring-integration-extensions'
}
developers {
developer {
id = 'not specified'
name = 'Soby Chacko'
email = 'not specified'
}
}
}
}
}

22
spring-integration-kafka/src/api/overview.html Normal file
View File

@@ -0,0 +1,22 @@
<html>
<body>
This document is the API specification for Spring Integration Kafka Extension
<hr/>
<div id="overviewBody">
<p>
For further API reference and developer documentation, see the
<a href="http://static.springsource.org/spring-integration/reference" target="_top">Spring
Integration reference documentation</a>.
That documentation contains more detailed, developer-targeted
descriptions, with conceptual overviews, definitions of terms,
workarounds, and working code examples.
</p>
<p>
If you are interested in commercial training, consultancy, and
support for Spring Integration, please visit <a href="http://www.springsource.com" target="_top">
http://www.springsource.com</a>
</p>
</div>
</body>
</html>

201
spring-integration-kafka/src/dist/license.txt vendored Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by testData1) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class testData1 and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [testData1 of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

21
spring-integration-kafka/src/dist/notice.txt vendored Normal file
View File

@@ -0,0 +1,21 @@
========================================================================
== NOTICE file corresponding to section 4 d of the Apache License, ==
== Version 2.0, in this case for the Spring Integration distribution. ==
========================================================================
This product includes software developed by
the Apache Software Foundation (http://www.apache.org).
The end-user documentation included with a redistribution, if any,
must include the following acknowledgement:
"This product includes software developed by the Spring Framework
Project (http://www.springframework.org)."
Alternatively, this acknowledgement may appear in the software itself,
if and wherever such third-party acknowledgements normally appear.
The names "Spring", "Spring Framework", and "Spring Integration" must
not be used to endorse or promote products derived from this software
without prior written permission. For written permission, please contact
enquiries@springsource.com.

122
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParser.java Normal file
View File

@@ -0,0 +1,122 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import org.springframework.beans.factory.config.BeanDefinition;
import org.springframework.beans.factory.config.BeanDefinitionHolder;
import org.springframework.beans.factory.support.AbstractBeanDefinition;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.xml.AbstractSingleBeanDefinitionParser;
import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.integration.kafka.support.ConsumerConfigFactoryBean;
import org.springframework.integration.kafka.support.ConsumerConfiguration;
import org.springframework.integration.kafka.support.ConsumerConnectionProvider;
import org.springframework.integration.kafka.support.ConsumerMetadata;
import org.springframework.integration.kafka.support.KafkaConsumerContext;
import org.springframework.integration.kafka.support.MessageLeftOverTracker;
import org.springframework.util.StringUtils;
import org.springframework.util.xml.DomUtils;
import org.w3c.dom.Element;
import java.util.HashMap;
import java.util.Map;
/**
* @author Soby Chacko
*/
public class KafkaConsumerContextParser extends AbstractSingleBeanDefinitionParser {
@Override
protected Class<?> getBeanClass(final Element element) {
return KafkaConsumerContext.class;
}
@Override
protected void doParse(final Element element, final ParserContext parserContext, final BeanDefinitionBuilder builder) {
super.doParse(element, parserContext, builder);
final Element consumerConfigurations = DomUtils.getChildElementByTagName(element, "consumer-configurations");
parseConsumerConfigurations(consumerConfigurations, parserContext, builder, element);
}
private void parseConsumerConfigurations(final Element consumerConfigurations, final ParserContext parserContext,
final BeanDefinitionBuilder builder, final Element parentElem) {
for (final Element consumerConfiguration : DomUtils.getChildElementsByTagName(consumerConfigurations, "consumer-configuration")) {
final BeanDefinitionBuilder consumerConfigurationBuilder = BeanDefinitionBuilder.genericBeanDefinition(ConsumerConfiguration.class);
final BeanDefinitionBuilder consumerMetadataBuilder = BeanDefinitionBuilder.genericBeanDefinition(ConsumerMetadata.class);
IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration, "group-id");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration, "value-decoder");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration, "key-decoder");
IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration, "key-class-type");
IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration, "value-class-type");
IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerConfigurationBuilder, consumerConfiguration, "max-messages");
IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerMetadataBuilder, parentElem, "consumer-timeout");
final Map<String, Integer> topicStreamsMap = new HashMap<String, Integer>();
for (final Element topicConfiguration : DomUtils.getChildElementsByTagName(consumerConfiguration, "topic")) {
final String topic = topicConfiguration.getAttribute("id");
final String streams = topicConfiguration.getAttribute("streams");
final Integer streamsInt = Integer.valueOf(streams);
topicStreamsMap.put(topic, streamsInt);
}
consumerMetadataBuilder.addPropertyValue("topicStreamMap", topicStreamsMap);
final BeanDefinition consumerMetadataBeanDef = consumerMetadataBuilder.getBeanDefinition();
registerBeanDefinition(new BeanDefinitionHolder(consumerMetadataBeanDef, "consumerMetadata_" + consumerConfiguration.getAttribute("group-id")),
parserContext.getRegistry());
final String zookeeperConnectBean = parentElem.getAttribute("zookeeper-connect");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, parentElem, zookeeperConnectBean);
final BeanDefinitionBuilder consumerConfigFactoryBuilder = BeanDefinitionBuilder.genericBeanDefinition(ConsumerConfigFactoryBean.class);
consumerConfigFactoryBuilder.addConstructorArgReference("consumerMetadata_" + consumerConfiguration.getAttribute("group-id"));
if (StringUtils.hasText(zookeeperConnectBean)) {
consumerConfigFactoryBuilder.addConstructorArgReference(zookeeperConnectBean);
}
final BeanDefinition consumerConfigFactoryBuilderBeanDefinition = consumerConfigFactoryBuilder.getBeanDefinition();
registerBeanDefinition(new BeanDefinitionHolder(consumerConfigFactoryBuilderBeanDefinition, "consumerConfigFactory_" + consumerConfiguration.getAttribute("group-id")), parserContext.getRegistry());
final BeanDefinitionBuilder consumerConnectionProviderBuilder = BeanDefinitionBuilder.genericBeanDefinition(ConsumerConnectionProvider.class);
consumerConnectionProviderBuilder.addConstructorArgReference("consumerConfigFactory_" + consumerConfiguration.getAttribute("group-id"));
final BeanDefinition consumerConnectionProviderBuilderBeanDefinition = consumerConnectionProviderBuilder.getBeanDefinition();
registerBeanDefinition(new BeanDefinitionHolder(consumerConnectionProviderBuilderBeanDefinition, "consumerConnectionProvider_" + consumerConfiguration.getAttribute("group-id")), parserContext.getRegistry());
final BeanDefinitionBuilder messageLeftOverBeanDefinitionBuilder = BeanDefinitionBuilder.genericBeanDefinition(MessageLeftOverTracker.class);
final BeanDefinition messageLeftOverBeanDefinition = messageLeftOverBeanDefinitionBuilder.getBeanDefinition();
registerBeanDefinition(new BeanDefinitionHolder(messageLeftOverBeanDefinition, "messageLeftOver_" + consumerConfiguration.getAttribute("group-id")),
parserContext.getRegistry());
consumerConfigurationBuilder.addConstructorArgReference("consumerMetadata_" + consumerConfiguration.getAttribute("group-id"));
consumerConfigurationBuilder.addConstructorArgReference("consumerConnectionProvider_" + consumerConfiguration.getAttribute("group-id"));
consumerConfigurationBuilder.addConstructorArgReference("messageLeftOver_" + consumerConfiguration.getAttribute("group-id"));
final AbstractBeanDefinition consumerConfigurationBeanDefinition = consumerConfigurationBuilder.getBeanDefinition();
final String consumerConfigurationBeanName = "consumerConfiguration_" + consumerConfiguration.getAttribute("group-id");
registerBeanDefinition(new BeanDefinitionHolder(consumerConfigurationBeanDefinition, consumerConfigurationBeanName),
parserContext.getRegistry());
}
}
}

49
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaInboundChannelAdapterParser.java Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import org.springframework.beans.BeanMetadataElement;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.integration.kafka.inbound.KafkaHighLevelConsumerMessageSource;
import org.springframework.util.StringUtils;
import org.w3c.dom.Element;
/**
* The Kafka Inbound Channel adapter parser
*
* @author Soby Chacko
*
*/
public class KafkaInboundChannelAdapterParser extends AbstractPollingInboundChannelAdapterParser {
@Override
protected BeanMetadataElement parseSource(final Element element, final ParserContext parserContext) {
final BeanDefinitionBuilder highLevelConsumerMessageSourceBuilder =
BeanDefinitionBuilder.genericBeanDefinition(KafkaHighLevelConsumerMessageSource.class);
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(highLevelConsumerMessageSourceBuilder, element, "kafka-decoder");
final String kafkaConsumerContext = element.getAttribute("kafka-consumer-context-ref");
if (StringUtils.hasText(kafkaConsumerContext)) {
highLevelConsumerMessageSourceBuilder.addConstructorArgReference(kafkaConsumerContext);
}
return highLevelConsumerMessageSourceBuilder.getBeanDefinition();
}
}

38
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaNamespaceHandler.java Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import org.springframework.integration.config.xml.AbstractIntegrationNamespaceHandler;
/**
* The namespace handler for the Kafka namespace
*
* @author Soby Chacko
*
*/
public class KafkaNamespaceHandler extends AbstractIntegrationNamespaceHandler {
/* (non-Javadoc)
* @see org.springframework.beans.factory.xml.NamespaceHandler#init()
*/
@Override
public void init() {
registerBeanDefinitionParser("zookeeper-connect", new ZookeeperConnectParser());
registerBeanDefinitionParser("inbound-channel-adapter", new KafkaInboundChannelAdapterParser());
registerBeanDefinitionParser("outbound-channel-adapter", new KafkaOutboundChannelAdapterParser());
registerBeanDefinitionParser("producer-context", new KafkaProducerContextParser());
registerBeanDefinitionParser("consumer-context", new KafkaConsumerContextParser());
}
}

45
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaOutboundChannelAdapterParser.java Normal file
View File

@@ -0,0 +1,45 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import org.springframework.beans.factory.support.AbstractBeanDefinition;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.xml.AbstractOutboundChannelAdapterParser;
import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler;
import org.springframework.util.StringUtils;
import org.w3c.dom.Element;
/**
*
* @author Soby Chacko
*
*/
public class KafkaOutboundChannelAdapterParser extends AbstractOutboundChannelAdapterParser {
@Override
protected AbstractBeanDefinition parseConsumer(final Element element, final ParserContext parserContext) {
final BeanDefinitionBuilder kafkaProducerMessageHandlerBuilder =
BeanDefinitionBuilder.genericBeanDefinition(KafkaProducerMessageHandler.class);
final String kafkaServerBeanName = element.getAttribute("kafka-producer-context-ref");
if (StringUtils.hasText(kafkaServerBeanName)) {
kafkaProducerMessageHandlerBuilder.addConstructorArgReference(kafkaServerBeanName);
}
return kafkaProducerMessageHandlerBuilder.getBeanDefinition();
}
}

89
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParser.java Normal file
View File

@@ -0,0 +1,89 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import org.springframework.beans.factory.config.BeanDefinition;
import org.springframework.beans.factory.config.BeanDefinitionHolder;
import org.springframework.beans.factory.support.AbstractBeanDefinition;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.xml.AbstractSimpleBeanDefinitionParser;
import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.integration.kafka.support.KafkaProducerContext;
import org.springframework.integration.kafka.support.ProducerConfiguration;
import org.springframework.integration.kafka.support.ProducerFactoryBean;
import org.springframework.integration.kafka.support.ProducerMetadata;
import org.springframework.util.StringUtils;
import org.springframework.util.xml.DomUtils;
import org.w3c.dom.Element;
/**
* @author Soby Chacko
*/
public class KafkaProducerContextParser extends AbstractSimpleBeanDefinitionParser {
@Override
protected Class<?> getBeanClass(final Element element) {
return KafkaProducerContext.class;
}
@Override
protected void doParse(final Element element, final ParserContext parserContext, final BeanDefinitionBuilder builder) {
super.doParse(element, parserContext, builder);
final Element topics = DomUtils.getChildElementByTagName(element, "producer-configurations");
parseProducerConfigurations(topics, parserContext);
}
private void parseProducerConfigurations(final Element topics, final ParserContext parserContext) {
for (final Element producerConfiguration : DomUtils.getChildElementsByTagName(topics, "producer-configuration")){
final BeanDefinitionBuilder producerConfigurationBuilder = BeanDefinitionBuilder.genericBeanDefinition(ProducerConfiguration.class);
final BeanDefinitionBuilder producerMetadataBuilder = BeanDefinitionBuilder.genericBeanDefinition(ProducerMetadata.class);
producerMetadataBuilder.addConstructorArgValue(producerConfiguration.getAttribute("topic"));
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "value-encoder");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "key-encoder");
IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "key-class-type");
IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "value-class-type");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "partitioner");
IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "compression-codec");
IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "async");
IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "batch-num-messages");
final BeanDefinition producerMetadataBeanDef = producerMetadataBuilder.getBeanDefinition();
registerBeanDefinition(new BeanDefinitionHolder(producerMetadataBeanDef, "producerMetadata_" + producerConfiguration.getAttribute("topic")),
parserContext.getRegistry());
final BeanDefinitionBuilder producerFactoryBuilder = BeanDefinitionBuilder.genericBeanDefinition(ProducerFactoryBean.class);
producerFactoryBuilder.addConstructorArgReference("producerMetadata_" + producerConfiguration.getAttribute("topic"));
final String brokerList = producerConfiguration.getAttribute("broker-list");
if (StringUtils.hasText(brokerList)) {
producerFactoryBuilder.addConstructorArgValue(producerConfiguration.getAttribute("broker-list"));
}
final BeanDefinition producerfactoryBeanDefinition = producerFactoryBuilder.getBeanDefinition();
registerBeanDefinition(new BeanDefinitionHolder(producerfactoryBeanDefinition, "prodFactory_" + producerConfiguration.getAttribute("topic")), parserContext.getRegistry());
producerConfigurationBuilder.addConstructorArgReference("producerMetadata_" + producerConfiguration.getAttribute("topic"));
producerConfigurationBuilder.addConstructorArgReference("prodFactory_" + producerConfiguration.getAttribute("topic"));
final AbstractBeanDefinition producerConfigurationBeanDefinition = producerConfigurationBuilder.getBeanDefinition();
final String producerConfigurationBeanName = "producerConfiguration_" + producerConfiguration.getAttribute("topic");
registerBeanDefinition(new BeanDefinitionHolder(producerConfigurationBeanDefinition, producerConfigurationBeanName),
parserContext.getRegistry());
}
}
}

46
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParser.java Normal file
View File

@@ -0,0 +1,46 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.xml.AbstractSimpleBeanDefinitionParser;
import org.springframework.beans.factory.xml.BeanDefinitionParserDelegate;
import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.integration.kafka.support.ZookeeperConnect;
import org.w3c.dom.Element;
/**
* @author Soby Chacko
*/
public class ZookeeperConnectParser extends AbstractSimpleBeanDefinitionParser {
@Override
protected Class<?> getBeanClass(final Element element) {
return ZookeeperConnect.class;
}
@Override
protected void doParse(final Element element, final ParserContext parserContext, final BeanDefinitionBuilder builder) {
super.doParse(element, parserContext, builder);
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element,
BeanDefinitionParserDelegate.SCOPE_ATTRIBUTE);
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-connect");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-connection-timeout");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-session-timeout");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-sync-time");
}
}

4
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/package-info.java Normal file
View File

@@ -0,0 +1,4 @@
/**
* Provides parser classes to provide Xml namespace support for the Kafka components.
*/
package org.springframework.integration.kafka.config.xml;

40
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java Normal file
View File

@@ -0,0 +1,40 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
/**
* Kafka adapter specific message headers.
*
* @author Soby Chacko
*/
public class KafkaConsumerDefaults {
//High level consumer
public static final String GROUP_ID = "groupid";
public static final String SOCKET_TIMEOUT = "30000";
public static final String SOCKET_BUFFER_SIZE = "64*1024";
public static final String FETCH_SIZE = "300 * 1024";
public static final String BACKOFF_INCREMENT = "1000";
public static final String QUEUED_CHUNKS_MAX = "100";
public static final String AUTO_COMMIT_ENABLE = "true";
public static final String AUTO_COMMIT_INTERVAL = "10000";
public static final String AUTO_OFFSET_RESET = "smallest";
//Overriding the default value of -1, which will make the consumer to wait indefinitely
public static final String CONSUMER_TIMEOUT = "5000";
public static final String REBALANCE_RETRIES_MAX = "4";
private KafkaConsumerDefaults() {
}
}

31
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConnectDefaults.java Normal file
View File

@@ -0,0 +1,31 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.core;
/**
*
* @author Soby Chacko
*
*/
public class ZookeeperConnectDefaults {
public static final String ZK_CONNECT = "localhost:2181";
public static final String ZK_CONNECTION_TIMEOUT = "6000";
public static final String ZK_SESSION_TIMEOUT = "6000";
public static final String ZK_SYNC_TIME = "2000";
private ZookeeperConnectDefaults() {
}
}

4
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/package-info.java Normal file
View File

@@ -0,0 +1,4 @@
/**
* Provides core classes of the Kafka module.
*/
package org.springframework.integration.kafka.core;

47
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaHighLevelConsumerMessageSource.java Normal file
View File

@@ -0,0 +1,47 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.inbound;
import org.springframework.integration.Message;
import org.springframework.integration.context.IntegrationObjectSupport;
import org.springframework.integration.core.MessageSource;
import org.springframework.integration.kafka.support.KafkaConsumerContext;
import java.util.List;
import java.util.Map;
/**
* @author Soby Chacko
*
*/
public class KafkaHighLevelConsumerMessageSource extends IntegrationObjectSupport implements MessageSource<Map<String, Map<Integer, List<Object>>>> {
private final KafkaConsumerContext kafkaConsumerContext;
public KafkaHighLevelConsumerMessageSource(final KafkaConsumerContext kafkaConsumerContext) {
this.kafkaConsumerContext = kafkaConsumerContext;
}
@Override
public Message<Map<String, Map<Integer, List<Object>>>> receive() {
return kafkaConsumerContext.receive();
}
@Override
public String getComponentType() {
return "kafka:inbound-channel-adapter";
}
}

4
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/package-info.java Normal file
View File

@@ -0,0 +1,4 @@
/**
* Provides inbound Spring Integration Kafka components.
*/
package org.springframework.integration.kafka.inbound;

41
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/outbound/KafkaProducerMessageHandler.java Normal file
View File

@@ -0,0 +1,41 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.outbound;
import org.springframework.integration.Message;
import org.springframework.integration.handler.AbstractMessageHandler;
import org.springframework.integration.kafka.support.KafkaProducerContext;
/**
* @author Soby Chacko
*/
public class KafkaProducerMessageHandler extends AbstractMessageHandler {
private final KafkaProducerContext kafkaProducerContext;
public KafkaProducerMessageHandler(final KafkaProducerContext kafkaProducerContext) {
this.kafkaProducerContext = kafkaProducerContext;
}
public KafkaProducerContext getKafkaProducerContext() {
return kafkaProducerContext;
}
@Override
protected void handleMessageInternal(final Message<?> message) throws Exception {
kafkaProducerContext.send(message);
}
}

4
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/package-info.java Normal file
View File

@@ -0,0 +1,4 @@
/**
* Root package of the Kafka Module.
*/
package org.springframework.integration.kafka;

54
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroBackedKafkaDecoder.java Normal file
View File

@@ -0,0 +1,54 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.serializer.avro;
import kafka.serializer.Decoder;
import org.apache.avro.Schema;
import org.apache.avro.reflect.ReflectData;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* @author Soby Chacko
*/
public class AvroBackedKafkaDecoder<T> implements Decoder<T> {
private static final Log LOG = LogFactory.getLog(AvroBackedKafkaDecoder.class);
private final Class clazz;
public AvroBackedKafkaDecoder(final Class clazz) {
this.clazz = clazz;
}
@Override
@SuppressWarnings("unchecked")
public T fromBytes(final byte[] bytes) {
final Schema schema = ReflectData.get().getSchema(clazz);
final AvroSerializer avroSerializer = new AvroSerializer();
try {
return (T) avroSerializer.deserialize(bytes, schema);
} catch (IOException e) {
LOG.error("Failed to decode byte array for schema: " + schema.getFullName(), e);
}
return null;
}
}

52
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroBackedKafkaEncoder.java Normal file
View File

@@ -0,0 +1,52 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.serializer.avro;
import kafka.serializer.Encoder;
import org.apache.avro.Schema;
import org.apache.avro.reflect.ReflectData;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* @author Soby Chacko
*/
public class AvroBackedKafkaEncoder<T> implements Encoder<T> {
private static final Log LOG = LogFactory.getLog(AvroBackedKafkaEncoder.class);
private final Class clazz;
public AvroBackedKafkaEncoder(final Class clazz) {
this.clazz = clazz;
}
@Override
@SuppressWarnings("unchecked")
public byte[] toBytes(final T source) {
final Schema schema = ReflectData.get().getSchema(clazz);
final AvroSerializer avroSerializer = new AvroSerializer();
try {
return avroSerializer.serialize(source, schema);
} catch (IOException e) {
LOG.error("Failed to encode source for schema: " + schema.getFullName());
}
return null;
}
}

52
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSerializer.java Normal file
View File

@@ -0,0 +1,52 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.serializer.avro;
import org.apache.avro.Schema;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.Encoder;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.reflect.ReflectDatumWriter;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
/**
* @author Soby Chacko
*/
public class AvroSerializer<T> {
public T deserialize(final byte[] bytes, final Schema schema) throws IOException {
final Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null);
final DatumReader<T> reader = new ReflectDatumReader<T>(schema);
return reader.read(null, decoder);
}
public byte[] serialize(final T input, final Schema schema) throws IOException {
final DatumWriter<T> writer = new ReflectDatumWriter<T>(schema);
final ByteArrayOutputStream stream = new ByteArrayOutputStream();
final Encoder encoder = EncoderFactory.get().binaryEncoder(stream, null);
writer.write(input, encoder);
encoder.flush();
return stream.toByteArray();
}
}

52
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumSerializer.java Normal file
View File

@@ -0,0 +1,52 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.serializer.avro;
import org.apache.avro.Schema;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.Encoder;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.specific.SpecificDatumWriter;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
/**
* @author Soby Chacko
*/
public class AvroSpecificDatumSerializer<T> {
public T deserialize(final byte[] bytes, final Schema schema) throws IOException {
final Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null);
final DatumReader<T> reader = new SpecificDatumReader<T>(schema);
return reader.read(null, decoder);
}
public byte[] serialize(final T input, final Schema schema) throws IOException {
final DatumWriter<T> writer = new SpecificDatumWriter<T>(schema);
final ByteArrayOutputStream stream = new ByteArrayOutputStream();
final Encoder encoder = EncoderFactory.get().binaryEncoder(stream, null);
writer.write(input, encoder);
encoder.flush();
return stream.toByteArray();
}
}

41
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringEncoder.java Normal file
View File

@@ -0,0 +1,41 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.serializer.common;
import kafka.serializer.Encoder;
import kafka.utils.VerifiableProperties;
import java.util.Properties;
/**
* @author Soby Chacko
*/
public class StringEncoder implements Encoder {
private String encoding = "UTF8";
public void setEncoding(final String encoding){
this.encoding = encoding;
}
@Override
public byte[] toBytes(final Object o) {
final Properties props = new Properties();
props.put("serializer.encoding", encoding);
final VerifiableProperties verifiableProperties = new VerifiableProperties(props);
return new kafka.serializer.StringEncoder(verifiableProperties).toBytes((String)o);
}
}

59
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfigFactoryBean.java Normal file
View File

@@ -0,0 +1,59 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import kafka.consumer.ConsumerConfig;
import org.springframework.beans.factory.FactoryBean;
import java.util.Properties;
/**
* @author Soby Chacko
*/
public class ConsumerConfigFactoryBean implements FactoryBean<ConsumerConfig> {
private final ConsumerMetadata consumerMetadata;
private final ZookeeperConnect zookeeperConnect;
public ConsumerConfigFactoryBean(final ConsumerMetadata consumerMetadata,
final ZookeeperConnect zookeeperConnect){
this.consumerMetadata = consumerMetadata;
this.zookeeperConnect = zookeeperConnect;
}
@Override
public ConsumerConfig getObject() throws Exception {
final Properties properties = new Properties();
properties.put("zookeeper.connect", zookeeperConnect.getZkConnect());
properties.put("zookeeper.session.timeout.ms", zookeeperConnect.getZkSessionTimeout());
properties.put("zookeeper.sync.time.ms", zookeeperConnect.getZkSyncTime());
properties.put("auto.commit.interval.ms", consumerMetadata.getAutoCommitInterval());
properties.put("consumer.timeout.ms", consumerMetadata.getConsumerTimeout());
properties.put("group.id", consumerMetadata.getGroupId());
return new ConsumerConfig(properties);
}
@Override
public Class<?> getObjectType() {
return ConsumerConfig.class;
}
@Override
public boolean isSingleton() {
return true;
}
}

209
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfiguration.java Normal file
View File

@@ -0,0 +1,209 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import kafka.consumer.ConsumerTimeoutException;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.message.MessageAndMetadata;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.integration.MessagingException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/**
* @author Soby Chacko
*/
public class ConsumerConfiguration {
private static final Log LOGGER = LogFactory.getLog(ConsumerConfiguration.class);
private final ConsumerMetadata consumerMetadata;
private final ConsumerConnectionProvider consumerConnectionProvider;
private final MessageLeftOverTracker messageLeftOverTracker;
private ConsumerConnector consumerConnector;
private volatile int count = 0;
private int maxMessages = 1;
private ExecutorService executorService = Executors.newCachedThreadPool();
public ConsumerConfiguration(final ConsumerMetadata consumerMetadata,
final ConsumerConnectionProvider consumerConnectionProvider,
final MessageLeftOverTracker messageLeftOverTracker) {
this.consumerMetadata = consumerMetadata;
this.consumerConnectionProvider = consumerConnectionProvider;
this.messageLeftOverTracker = messageLeftOverTracker;
}
public ConsumerMetadata getConsumerMetadata() {
return consumerMetadata;
}
public Map<String, Map<Integer, List<Object>>> receive() {
count = messageLeftOverTracker.getCurrentCount();
final List<Callable<List<MessageAndMetadata>>> tasks = new LinkedList<Callable<List<MessageAndMetadata>>>();
final Object lock = new Object();
final Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = getConsumerMapWithMessageStreams();
for (final List<KafkaStream<byte[], byte[]>> streams : consumerMap.values()) {
for (final KafkaStream<byte[], byte[]> stream : streams) {
tasks.add(new Callable<List<MessageAndMetadata>>() {
@Override
public List<MessageAndMetadata> call() throws Exception {
final List<MessageAndMetadata> rawMessages = new ArrayList<MessageAndMetadata>();
try {
while (count < maxMessages) {
final MessageAndMetadata messageAndMetadata = stream.iterator().next();
synchronized (lock) {
if (count < maxMessages) {
rawMessages.add(messageAndMetadata);
count++;
} else {
messageLeftOverTracker.addMessageAndMetadata(messageAndMetadata);
}
}
}
} catch (ConsumerTimeoutException cte) {
LOGGER.info("Consumer timed out");
}
return rawMessages;
}
});
}
}
return executeTasks(tasks);
}
private Map<String, Map<Integer, List<Object>>> executeTasks(final List<Callable<List<MessageAndMetadata>>> tasks) {
final Map<String, Map<Integer, List<Object>>> messages = new ConcurrentHashMap<String, Map<Integer, List<Object>>>();
messages.putAll(getLeftOverMessageMap());
try {
for (final Future<List<MessageAndMetadata>> result : executorService.invokeAll(tasks)) {
if (!result.get().isEmpty()) {
final String topic = result.get().get(0).topic();
if (!messages.containsKey(topic)) {
messages.put(topic, getPayload(result.get()));
} else {
final Map<Integer, List<Object>> existingPayloadMap = messages.get(topic);
getPayload(result.get(), existingPayloadMap);
}
}
}
} catch (Exception e) {
throw new MessagingException("Consuming from Kafka failed", e);
}
if (messages.isEmpty()) {
return null;
}
return messages;
}
private Map<String, Map<Integer, List<Object>>> getLeftOverMessageMap() {
final Map<String, Map<Integer, List<Object>>> messages = new ConcurrentHashMap<String, Map<Integer, List<Object>>>();
for (final MessageAndMetadata mamd : messageLeftOverTracker.getMessageLeftOverFromPreviousPoll()) {
final String topic = mamd.topic();
if (!messages.containsKey(topic)) {
final List<MessageAndMetadata> l = new ArrayList<MessageAndMetadata>();
l.add(mamd);
messages.put(topic, getPayload(l));
} else {
final Map<Integer, List<Object>> existingPayloadMap = messages.get(topic);
final List<MessageAndMetadata> l = new ArrayList<MessageAndMetadata>();
l.add(mamd);
getPayload(l, existingPayloadMap);
}
}
messageLeftOverTracker.clearMessagesLeftOver();
return messages;
}
private Map<Integer, List<Object>> getPayload(final List<MessageAndMetadata> messageAndMetadatas) {
final Map<Integer, List<Object>> payloadMap = new ConcurrentHashMap<Integer, List<Object>>();
for (final MessageAndMetadata messageAndMetadata : messageAndMetadatas) {
if (!payloadMap.containsKey(messageAndMetadata.partition())) {
final List<Object> payload = new ArrayList<Object>();
payload.add(messageAndMetadata.message());
payloadMap.put(messageAndMetadata.partition(), payload);
} else {
final List<Object> payload = payloadMap.get(messageAndMetadata.partition());
payload.add(messageAndMetadata.message());
}
}
return payloadMap;
}
private void getPayload(final List<MessageAndMetadata> messageAndMetadatas, final Map<Integer, List<Object>> existingPayloadMap) {
for (final MessageAndMetadata messageAndMetadata : messageAndMetadatas) {
if (!existingPayloadMap.containsKey(messageAndMetadata.partition())) {
final List<Object> payload = new ArrayList<Object>();
payload.add(messageAndMetadata.message());
existingPayloadMap.put(messageAndMetadata.partition(), payload);
} else {
final List<Object> payload = existingPayloadMap.get(messageAndMetadata.partition());
payload.add(messageAndMetadata.message());
}
}
}
@SuppressWarnings("unchecked")
public Map<String, List<KafkaStream<byte[], byte[]>>> getConsumerMapWithMessageStreams() {
if (consumerMetadata.getValueDecoder() != null) {
return getConsumerConnector().createMessageStreams(
consumerMetadata.getTopicStreamMap(),
consumerMetadata.getValueDecoder(),
consumerMetadata.getValueDecoder());
}
return getConsumerConnector().createMessageStreams(consumerMetadata.getTopicStreamMap());
}
public int getMaxMessages() {
return maxMessages;
}
public void setMaxMessages(final int maxMessages) {
this.maxMessages = maxMessages;
}
public ConsumerConnector getConsumerConnector() {
if (consumerConnector == null) {
consumerConnector = consumerConnectionProvider.getConsumerConnector();
}
return consumerConnector;
}
}

35
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConnectionProvider.java Normal file
View File

@@ -0,0 +1,35 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import kafka.consumer.ConsumerConfig;
import kafka.javaapi.consumer.ConsumerConnector;
/**
* @author Soby Chacko
*/
public class ConsumerConnectionProvider {
private final ConsumerConfig consumerConfig;
public ConsumerConnectionProvider(final ConsumerConfig consumerConfig) {
this.consumerConfig = consumerConfig;
}
public ConsumerConnector getConsumerConnector() {
return kafka.consumer.Consumer.createJavaConsumerConnector(consumerConfig);
}
}

174
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerMetadata.java Normal file
View File

@@ -0,0 +1,174 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import kafka.serializer.Decoder;
import org.springframework.integration.kafka.core.KafkaConsumerDefaults;
import java.util.Map;
/**
* @author Soby Chacko
*/
public class ConsumerMetadata<K,V> {
//High level consumer defaults
private String groupId = KafkaConsumerDefaults.GROUP_ID;
private String socketTimeout = KafkaConsumerDefaults.SOCKET_TIMEOUT;
private String socketBufferSize = KafkaConsumerDefaults.SOCKET_BUFFER_SIZE;
private String fetchSize = KafkaConsumerDefaults.FETCH_SIZE;
private String backoffIncrement = KafkaConsumerDefaults.BACKOFF_INCREMENT;
private String queuedChunksMax = KafkaConsumerDefaults.QUEUED_CHUNKS_MAX;
private String autoCommitEnable = KafkaConsumerDefaults.AUTO_COMMIT_ENABLE;
private String autoCommitInterval = KafkaConsumerDefaults.AUTO_COMMIT_INTERVAL;
private String autoOffsetReset = KafkaConsumerDefaults.AUTO_OFFSET_RESET;
private String rebalanceRetriesMax = KafkaConsumerDefaults.REBALANCE_RETRIES_MAX;
private String consumerTimeout = KafkaConsumerDefaults.CONSUMER_TIMEOUT;
private String topic;
private int streams;
private Decoder<V> valueDecoder;
private Decoder<K> keyDecoder;
private Map<String, Integer> topicStreamMap;
public String getGroupId() {
return groupId;
}
public void setGroupId(final String groupId) {
this.groupId = groupId;
}
public String getSocketTimeout() {
return socketTimeout;
}
public void setSocketTimeout(final String socketTimeout) {
this.socketTimeout = socketTimeout;
}
public String getSocketBufferSize() {
return socketBufferSize;
}
public void setSocketBufferSize(final String socketBufferSize) {
this.socketBufferSize = socketBufferSize;
}
public String getFetchSize() {
return fetchSize;
}
public void setFetchSize(final String fetchSize) {
this.fetchSize = fetchSize;
}
public String getBackoffIncrement() {
return backoffIncrement;
}
public void setBackoffIncrement(final String backoffIncrement) {
this.backoffIncrement = backoffIncrement;
}
public String getQueuedChunksMax() {
return queuedChunksMax;
}
public void setQueuedChunksMax(final String queuedChunksMax) {
this.queuedChunksMax = queuedChunksMax;
}
public String getAutoCommitEnable() {
return autoCommitEnable;
}
public void setAutoCommitEnable(final String autoCommitEnable) {
this.autoCommitEnable = autoCommitEnable;
}
public String getAutoCommitInterval() {
return autoCommitInterval;
}
public void setAutoCommitInterval(final String autoCommitInterval) {
this.autoCommitInterval = autoCommitInterval;
}
public String getAutoOffsetReset() {
return autoOffsetReset;
}
public void setAutoOffsetReset(final String autoOffsetReset) {
this.autoOffsetReset = autoOffsetReset;
}
public String getRebalanceRetriesMax() {
return rebalanceRetriesMax;
}
public void setRebalanceRetriesMax(final String rebalanceRetriesMax) {
this.rebalanceRetriesMax = rebalanceRetriesMax;
}
public String getConsumerTimeout() {
return consumerTimeout;
}
public void setConsumerTimeout(final String consumerTimeout) {
this.consumerTimeout = consumerTimeout;
}
public String getTopic() {
return topic;
}
public void setTopic(final String topic) {
this.topic = topic;
}
public int getStreams() {
return streams;
}
public void setStreams(final int streams) {
this.streams = streams;
}
public Decoder<V> getValueDecoder() {
return valueDecoder;
}
public void setValueDecoder(final Decoder<V> valueDecoder) {
this.valueDecoder = valueDecoder;
}
public Decoder<K> getKeyDecoder() {
return keyDecoder;
}
public void setKeyDecoder(final Decoder<K> keyDecoder) {
this.keyDecoder = keyDecoder;
}
public Map<String, Integer> getTopicStreamMap() {
return topicStreamMap;
}
public void setTopicStreamMap(final Map<String, Integer> topicStreamMap) {
this.topicStreamMap = topicStreamMap;
}
}

36
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultPartitioner.java Normal file
View File

@@ -0,0 +1,36 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import kafka.producer.Partitioner;
import kafka.utils.Utils;
/**
* @author Soby Chacko
*
* This class is for internal use only and therefore is at default access level
*/
class DefaultPartitioner<T> implements Partitioner<T> {
/**
* Uses the key to calculate a partition bucket id for routing
* the data to the appropriate broker partition
* @return an integer between 0 and numPartitions-1
*/
@Override
public int partition(final T key, final int numPartitions) {
return Utils.abs(key.hashCode()) % numPartitions;
}
}

76
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaConsumerContext.java Normal file
View File

@@ -0,0 +1,76 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.beans.factory.BeanFactoryAware;
import org.springframework.beans.factory.ListableBeanFactory;
import org.springframework.integration.Message;
import org.springframework.integration.kafka.core.KafkaConsumerDefaults;
import org.springframework.integration.support.MessageBuilder;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author Soby Chacko
*/
public class KafkaConsumerContext implements BeanFactoryAware {
private Map<String, ConsumerConfiguration> consumerConfigurations;
private String consumerTimeout = KafkaConsumerDefaults.CONSUMER_TIMEOUT;
private ZookeeperConnect zookeeperConnect;
public Collection<ConsumerConfiguration> getConsumerConfigurations() {
return consumerConfigurations.values();
}
@Override
public void setBeanFactory(final BeanFactory beanFactory) throws BeansException {
consumerConfigurations = ((ListableBeanFactory) beanFactory).getBeansOfType(ConsumerConfiguration.class);
}
public Message<Map<String, Map<Integer, List<Object>>>> receive() {
final Map<String, Map<Integer, List<Object>>> consumedData = new HashMap<String, Map<Integer, List<Object>>>();
for (final ConsumerConfiguration consumerConfiguration : getConsumerConfigurations()) {
final Map<String, Map<Integer, List<Object>>> messages = consumerConfiguration.receive();
if (messages != null){
consumedData.putAll(messages);
}
}
return MessageBuilder.withPayload(consumedData).build();
}
public String getConsumerTimeout() {
return consumerTimeout;
}
public void setConsumerTimeout(final String consumerTimeout) {
this.consumerTimeout = consumerTimeout;
}
public ZookeeperConnect getZookeeperConnect() {
return zookeeperConnect;
}
public void setZookeeperConnect(final ZookeeperConnect zookeeperConnect) {
this.zookeeperConnect = zookeeperConnect;
}
}

63
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaProducerContext.java Normal file
View File

@@ -0,0 +1,63 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.beans.factory.BeanFactoryAware;
import org.springframework.beans.factory.ListableBeanFactory;
import org.springframework.integration.Message;
import java.util.Collection;
import java.util.Map;
/**
* @author Soby Chacko
*/
public class KafkaProducerContext implements BeanFactoryAware {
private Map<String, ProducerConfiguration> topicsConfiguration;
@SuppressWarnings("unchecked")
public void send(final Message<?> message) throws Exception {
final ProducerConfiguration producerConfiguration =
getTopicConfiguration(message.getHeaders().get("topic", String.class));
if (producerConfiguration != null) {
producerConfiguration.send(message);
}
}
private ProducerConfiguration getTopicConfiguration(final String topic){
final Collection<ProducerConfiguration> topics = topicsConfiguration.values();
for (final ProducerConfiguration producerConfiguration : topics){
if (producerConfiguration.getProducerMetadata().getTopic().equals(topic)){
return producerConfiguration;
}
}
return null;
}
public Map<String, ProducerConfiguration> getTopicsConfiguration() {
return topicsConfiguration;
}
@Override
public void setBeanFactory(final BeanFactory beanFactory) throws BeansException {
topicsConfiguration = ((ListableBeanFactory)beanFactory).getBeansOfType(ProducerConfiguration.class);
}
}

44
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/MessageLeftOverTracker.java Normal file
View File

@@ -0,0 +1,44 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import kafka.message.MessageAndMetadata;
import java.util.ArrayList;
import java.util.List;
/**
* @author Soby Chacko
*/
public class MessageLeftOverTracker {
private final List<MessageAndMetadata> messageLeftOverFromPreviousPoll = new ArrayList<MessageAndMetadata>();
public void addMessageAndMetadata(final MessageAndMetadata messageAndMetadata){
messageLeftOverFromPreviousPoll.add(messageAndMetadata);
}
public List<MessageAndMetadata> getMessageLeftOverFromPreviousPoll(){
return messageLeftOverFromPreviousPoll;
}
public void clearMessagesLeftOver(){
messageLeftOverFromPreviousPoll.clear();
}
public int getCurrentCount() {
return messageLeftOverFromPreviousPoll.size();
}
}

102
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerConfiguration.java Normal file
View File

@@ -0,0 +1,102 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.serializer.DefaultEncoder;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.springframework.integration.Message;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
/**
* @author Soby Chacko
*/
public class ProducerConfiguration<K,V> {
private final Producer<K,V> producer;
private final ProducerMetadata<K,V> producerMetadata;
public ProducerConfiguration(final ProducerMetadata<K, V> producerMetadata, final Producer<K, V> producer){
this.producerMetadata = producerMetadata;
this.producer = producer;
}
public ProducerMetadata<K, V> getProducerMetadata() {
return producerMetadata;
}
public void send(final Message<?> message) throws Exception {
final V v = getPayload(message);
if (message.getHeaders().containsKey("messageKey")) {
producer.send(new KeyedMessage<K, V>(producerMetadata.getTopic(), getKey(message), v));
} else {
producer.send(new KeyedMessage<K, V>(producerMetadata.getTopic(), v));
}
}
@SuppressWarnings("unchecked")
private V getPayload(final Message<?> message) throws Exception {
if (producerMetadata.getValueEncoder().getClass().isAssignableFrom(DefaultEncoder.class)) {
return (V) getByteStream(message.getPayload());
} else if (message.getPayload().getClass().isAssignableFrom(producerMetadata.getValueClassType())) {
return producerMetadata.getValueClassType().cast(message.getPayload());
}
throw new Exception("Message payload type is not matching with what is configured");
}
@SuppressWarnings("unchecked")
private K getKey(final Message<?> message) throws Exception {
final Object key = message.getHeaders().get("messageKey");
if (producerMetadata.getKeyEncoder().getClass().isAssignableFrom(DefaultEncoder.class)) {
return (K) getByteStream(key);
}
return message.getHeaders().get("messageKey", producerMetadata.getKeyClassType());
}
private static boolean isRawByteArray(final Object obj){
return obj instanceof byte[];
}
private static byte[] getByteStream(final Object obj) throws IOException {
if (isRawByteArray(obj)){
return (byte[])obj;
}
final ByteArrayOutputStream out = new ByteArrayOutputStream();
final ObjectOutputStream os = new ObjectOutputStream(out);
os.writeObject(obj);
return out.toByteArray();
}
@Override
public boolean equals(final Object obj){
return EqualsBuilder.reflectionEquals(this, obj);
}
@Override
public int hashCode() {
return HashCodeBuilder.reflectionHashCode(this);
}
}

74
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerFactoryBean.java Normal file
View File

@@ -0,0 +1,74 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import kafka.javaapi.producer.Producer;
import kafka.producer.ProducerConfig;
import kafka.producer.ProducerPool;
import kafka.producer.async.DefaultEventHandler;
import kafka.producer.async.EventHandler;
import org.springframework.beans.factory.FactoryBean;
import scala.collection.mutable.HashMap;
import java.util.Properties;
/**
* @author Soby Chacko
*/
public class ProducerFactoryBean<K,V> implements FactoryBean<Producer<K,V>> {
private final String brokerList;
private final ProducerMetadata<K,V> producerMetadata;
public ProducerFactoryBean(final ProducerMetadata<K,V> producerMetadata, final String brokerList){
this.producerMetadata = producerMetadata;
this.brokerList = brokerList;
}
@Override
public Producer<K, V> getObject() throws Exception {
final Properties props = new Properties();
props.put("metadata.broker.list", brokerList);
props.put("compression.codec", producerMetadata.getCompressionCodec());
if (producerMetadata.isAsync()){
props.put("producer.type", "async");
if (producerMetadata.getBatchNumMessages() != null){
props.put("batch.num.messages", producerMetadata.getBatchNumMessages());
}
}
final ProducerConfig config = new ProducerConfig(props);
final EventHandler<K, V> eventHandler = new DefaultEventHandler<K, V>(config,
producerMetadata.getPartitioner() == null ? new DefaultPartitioner<K>() : producerMetadata.getPartitioner(),
producerMetadata.getValueEncoder(), producerMetadata.getKeyEncoder(),
new ProducerPool(config), new HashMap<String, kafka.api.TopicMetadata>());
final kafka.producer.Producer<K, V> prod = new kafka.producer.Producer<K, V>(config,
eventHandler);
return new Producer<K, V>(prod);
}
@Override
public Class<?> getObjectType() {
return Producer.class;
}
@Override
public boolean isSingleton() {
return true;
}
}

139
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerMetadata.java Normal file
View File

@@ -0,0 +1,139 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import kafka.producer.Partitioner;
import kafka.serializer.DefaultEncoder;
import kafka.serializer.Encoder;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.springframework.beans.factory.InitializingBean;
/**
* @author Soby Chacko
*/
public class ProducerMetadata<K,V> implements InitializingBean {
private Encoder<K> keyEncoder;
private Encoder<V> valueEncoder;
private Class<K> keyClassType;
private Class<V> valueClassType;
private final String topic;
private String compressionCodec = "default";
private Partitioner<K> partitioner;
private boolean async = false;
private String batchNumMessages;
public ProducerMetadata(final String topic) {
this.topic = topic;
}
public String getTopic() {
return topic;
}
public Encoder<K> getKeyEncoder() {
return keyEncoder;
}
public void setKeyEncoder(final Encoder<K> keyEncoder) {
this.keyEncoder = keyEncoder;
}
public Encoder<V> getValueEncoder() {
return valueEncoder;
}
public void setValueEncoder(final Encoder<V> valueEncoder) {
this.valueEncoder = valueEncoder;
}
public Class<K> getKeyClassType() {
return keyClassType;
}
public void setKeyClassType(final Class<K> keyClassType) {
this.keyClassType = keyClassType;
}
public Class<V> getValueClassType() {
return valueClassType;
}
public void setValueClassType(final Class<V> valueClassType) {
this.valueClassType = valueClassType;
}
//TODO: Use an enum
public String getCompressionCodec() {
if (compressionCodec.equalsIgnoreCase("gzip")) {
return "1";
} else if (compressionCodec.equalsIgnoreCase("snappy")) {
return "2";
}
return "0";
}
public void setCompressionCodec(final String compressionCodec) {
this.compressionCodec = compressionCodec;
}
public Partitioner<K> getPartitioner() {
return partitioner;
}
public void setPartitioner(final Partitioner<K> partitioner) {
this.partitioner = partitioner;
}
@Override
@SuppressWarnings("unchecked")
public void afterPropertiesSet() throws Exception {
if (valueEncoder == null) {
setValueEncoder((Encoder<V>) new DefaultEncoder(null));
}
if (keyEncoder == null) {
setKeyEncoder((Encoder<K>) getValueEncoder());
}
}
public boolean isAsync() {
return async;
}
public void setAsync(final boolean async) {
this.async = async;
}
public String getBatchNumMessages() {
return batchNumMessages;
}
public void setBatchNumMessages(final String batchNumMessages) {
this.batchNumMessages = batchNumMessages;
}
@Override
public boolean equals(final Object obj){
return EqualsBuilder.reflectionEquals(this, obj);
}
@Override
public int hashCode() {
return HashCodeBuilder.reflectionHashCode(this);
}
}

60
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ZookeeperConnect.java Normal file
View File

@@ -0,0 +1,60 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import org.springframework.integration.kafka.core.ZookeeperConnectDefaults;
/**
* @author Soby Chacko
*/
public class ZookeeperConnect {
private String zkConnect = ZookeeperConnectDefaults.ZK_CONNECT;
private String zkConnectionTimeout = ZookeeperConnectDefaults.ZK_CONNECTION_TIMEOUT;
private String zkSessionTimeout = ZookeeperConnectDefaults.ZK_SESSION_TIMEOUT;
private String zkSyncTime = ZookeeperConnectDefaults.ZK_SYNC_TIME;
public String getZkConnect() {
return zkConnect;
}
public void setZkConnect(final String zkConnect) {
this.zkConnect = zkConnect;
}
public String getZkConnectionTimeout() {
return zkConnectionTimeout;
}
public void setZkConnectionTimeout(final String zkConnectionTimeout) {
this.zkConnectionTimeout = zkConnectionTimeout;
}
public String getZkSessionTimeout() {
return zkSessionTimeout;
}
public void setZkSessionTimeout(final String zkSessionTimeout) {
this.zkSessionTimeout = zkSessionTimeout;
}
public String getZkSyncTime() {
return zkSyncTime;
}
public void setZkSyncTime(final String zkSyncTime) {
this.zkSyncTime = zkSyncTime;
}
}

4
spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/package-info.java Normal file
View File

@@ -0,0 +1,4 @@
/**
* Provides various support classes used across Spring Integration Kafka Components.
*/
package org.springframework.integration.kafka.support;

1
spring-integration-kafka/src/main/resources/META-INF/spring.handlers Normal file
View File

@@ -0,0 +1 @@
http\://www.springframework.org/schema/integration/kafka=org.springframework.integration.kafka.config.xml.KafkaNamespaceHandler

2
spring-integration-kafka/src/main/resources/META-INF/spring.schemas Normal file
View File

@@ -0,0 +1,2 @@
http\://www.springframework.org/schema/integration/kafka/spring-integration-kafka-1.0.xsd=org/springframework/integration/config/xml/spring-integration-kafka-1.0.xsd
http\://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd=org/springframework/integration/config/xml/spring-integration-kafka-1.0.xsd

4
spring-integration-kafka/src/main/resources/META-INF/spring.tooling Normal file
View File

@@ -0,0 +1,4 @@
# Tooling related information for the integration Kafka namespace
http\://www.springframework.org/schema/integration/kafka@name=integration Kafka Namespace
http\://www.springframework.org/schema/integration/kafka@prefix=int-kafka
http\://www.springframework.org/schema/integration/kafka@icon=org/springframework/integration/config/xml/spring-integration-kafka.gif

451
spring-integration-kafka/src/main/resources/org/springframework/integration/config/xml/spring-integration-kafka-1.0.xsd Normal file
View File

@@ -0,0 +1,451 @@
<?xml version="1.0" encoding="UTF-8"?>
<xsd:schema xmlns="http://www.springframework.org/schema/integration/kafka"
xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:beans="http://www.springframework.org/schema/beans"
xmlns:tool="http://www.springframework.org/schema/tool"
xmlns:integration="http://www.springframework.org/schema/integration"
targetNamespace="http://www.springframework.org/schema/integration/kafka"
elementFormDefault="qualified" attributeFormDefault="unqualified">
<xsd:import namespace="http://www.springframework.org/schema/beans"/>
<xsd:import namespace="http://www.springframework.org/schema/tool"/>
<xsd:import namespace="http://www.springframework.org/schema/integration"
schemaLocation="http://www.springframework.org/schema/integration/spring-integration.xsd"/>
<xsd:annotation>
<xsd:documentation><![CDATA[
Defines the configuration elements for the Spring Integration
Kafka Adapter.
]]></xsd:documentation>
</xsd:annotation>
<xsd:element name="zookeeper-connect">
<xsd:annotation>
<xsd:documentation><![CDATA[
Defines a Kafka server information.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:attribute name="id" type="xsd:string" use="required"/>
<xsd:attribute name="zk-connect" use="optional">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the Kafka server URL
]]></xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="direct">
<tool:expected-type type="java.lang.String"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="zk-connection-timeout" use="optional">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the Kafka consumer zkConnectionTimeout value
]]></xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="direct">
<tool:expected-type type="java.lang.Integer"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="zk-session-timeout" use="optional">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the Kafka consumer group id
]]></xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="direct">
<tool:expected-type type="java.lang.String"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="zk-sync-time" use="optional">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the Kafka consumer group id
]]></xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="direct">
<tool:expected-type type="java.lang.String"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="auto-commit-interval" use="optional">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the Kafka consumer group id
]]></xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="direct">
<tool:expected-type type="java.lang.String"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
</xsd:complexType>
</xsd:element>
<xsd:element name="producer-context">
<xsd:annotation>
<xsd:documentation><![CDATA[
Defines a producer context.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:sequence>
<xsd:element name="producer-configurations" minOccurs="0" maxOccurs="1">
<xsd:annotation>
<xsd:documentation><![CDATA[
Groups kafka topic configurations.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:choice>
<xsd:element name="producer-configuration" maxOccurs="unbounded">
<xsd:annotation>
<xsd:documentation><![CDATA[
Declares a kafka topic configuration which drives how and where a topic is sent to broker/s.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:attribute name="topic" type="xsd:string" use="required">
<xsd:annotation>
<xsd:documentation>
The topic configured by this configuration.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="broker-list" use="required">
<xsd:annotation>
<xsd:documentation><![CDATA[
list of comma separated kafka brokers.
]]></xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="direct">
<tool:expected-type type="java.lang.String"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="value-encoder" use="optional" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Custom implemenation of a Kafka Encoder for encoding message values.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="key-encoder" use="optional"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Custom implemenation of a Kafka Encoder for encoding message keys.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="key-class-type" use="optional"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Class type used for the key
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="value-class-type" use="optional"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Class type used for the value
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="compression-codec" use="optional">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the type of compression codec used for message compression.
]]></xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="direct">
<tool:expected-type type="java.lang.String"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="partitioner" use="optional"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Custom Kafka key partitioner.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="async" use="optional"
type="xsd:boolean">
<xsd:annotation>
<xsd:documentation>
Indicates if this producer is async or not.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="batch-num-messages" use="optional"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
number of messages to batch at this producer.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
</xsd:complexType>
</xsd:element>
</xsd:choice>
</xsd:complexType>
</xsd:element>
</xsd:sequence>
<xsd:attribute name="id" type="xsd:string" use="required"/>
</xsd:complexType>
</xsd:element>
<xsd:element name="consumer-context">
<xsd:annotation>
<xsd:documentation><![CDATA[
Defines a producer context.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:sequence>
<xsd:element name="consumer-configurations" minOccurs="0" maxOccurs="1">
<xsd:annotation>
<xsd:documentation><![CDATA[
Groups kafka topic configurations.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:choice>
<xsd:element name="consumer-configuration" maxOccurs="unbounded">
<xsd:annotation>
<xsd:documentation><![CDATA[
Declares a kafka topic configuration which drives how and where a topic is sent to broker/s.
]]></xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:sequence>
<xsd:element name="topic" maxOccurs="unbounded">
<xsd:complexType>
<xsd:attribute name="id" type="xsd:string" use="required"/>
<xsd:attribute name="streams" type="xsd:string" use="required"/>
</xsd:complexType>
</xsd:element>
</xsd:sequence>
<xsd:attribute name="max-messages" use="optional">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates max messages to aggregate in a single call to receive
]]></xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="direct">
<tool:expected-type type="java.lang.String"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="group-id" use="required">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the Kafka consumer group id
]]></xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="direct">
<tool:expected-type type="java.lang.String"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="value-decoder" use="optional"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Kafka Server Bean Name
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="key-decoder" use="optional"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Kafka Server Bean Name
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
</xsd:complexType>
</xsd:element>
</xsd:choice>
</xsd:complexType>
</xsd:element>
</xsd:sequence>
<xsd:attribute name="id" type="xsd:string" use="required"/>
<xsd:attribute name="consumer-timeout" use="optional">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the Kafka consumer timeout ms
]]></xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="direct">
<tool:expected-type type="java.lang.String"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="zookeeper-connect" use="required"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Kafka Server Bean Name
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
</xsd:complexType>
</xsd:element>
<xsd:element name="inbound-channel-adapter">
<xsd:annotation>
<xsd:documentation>
The definition for the Spring Integration Kafka
Inbound Channel Adapter.
</xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:sequence>
<xsd:element ref="integration:poller" minOccurs="0" maxOccurs="1"/>
</xsd:sequence>
<xsd:attributeGroup ref="coreKafkaComponentAttributes"/>
<xsd:attribute name="channel" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.integration.core.MessageChannel"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="send-timeout" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[
Allows you to specify how long this inbound-channel-adapter
will wait for the message (containing the retrieved entities)
to be sent successfully to the message channel, before throwing
an exception.
Keep in mind that when sending to a DirectChannel, the
invocation will occur in the sender's thread so the failing
of the send operation may be caused by other components
further downstream. By default the Inbound Channel Adapter
will wait indefinitely. The value is specified in milliseconds.
]]>
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="group-id" use="optional">
<xsd:annotation>
<xsd:documentation><![CDATA[
Indicates the Kafka consumer group id
]]></xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="direct">
<tool:expected-type type="java.lang.String"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="kafka-consumer-context-ref" use="required"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Kafka Server Bean Name
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
</xsd:complexType>
</xsd:element>
<xsd:attributeGroup name="coreKafkaComponentAttributes">
<xsd:attribute name="id" type="xsd:string" use="optional">
<xsd:annotation>
<xsd:documentation>
Identifies the underlying Spring bean definition, which is an
instance of either 'EventDrivenConsumer' or 'PollingConsumer',
depending on whether the component's input channel is a
'SubscribableChannel' or 'PollableChannel'.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="auto-startup" default="true" use="optional">
<xsd:annotation>
<xsd:documentation>
Flag to indicate that the component should start automatically
on startup (default true).
</xsd:documentation>
</xsd:annotation>
<xsd:simpleType>
<xsd:union memberTypes="xsd:boolean xsd:string"/>
</xsd:simpleType>
</xsd:attribute>
</xsd:attributeGroup>
<xsd:element name="outbound-channel-adapter">
<xsd:annotation>
<xsd:documentation>
Defines kafka outbound channel adapter that writes the contents of the
Message to kafka broker.
</xsd:documentation>
</xsd:annotation>
<xsd:complexType>
<xsd:sequence>
<xsd:element ref="integration:poller" minOccurs="0" maxOccurs="1"/>
</xsd:sequence>
<xsd:attribute name="id" type="xsd:string" use="optional">
<xsd:annotation>
<xsd:documentation>
Identifies the underlying Spring bean definition, which is an
instance of either 'EventDrivenConsumer' or 'PollingConsumer',
depending on whether the component's input channel is a
'SubscribableChannel' or 'PollableChannel'.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="auto-startup" default="true" use="optional">
<xsd:annotation>
<xsd:documentation>
Flag to indicate that the component should start automatically
on startup (default true).
</xsd:documentation>
</xsd:annotation>
<xsd:simpleType>
<xsd:union memberTypes="xsd:boolean xsd:string"/>
</xsd:simpleType>
</xsd:attribute>
<xsd:attribute name="kafka-producer-context-ref" use="required"
type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Kafka producer context reference.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="channel" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.integration.core.MessageChannel"/>
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
</xsd:complexType>
</xsd:element>
</xsd:schema>

BIN
spring-integration-kafka/src/main/resources/org/springframework/integration/config/xml/spring-integration-kafka.gif Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 572 B

72
spring-integration-kafka/src/reference/docbook/SIAdapterLowerPrefix.xml Normal file
View File

@@ -0,0 +1,72 @@
<?xml version="1.0" encoding="UTF-8"?>
<chapter xmlns="http://docbook.org/ns/docbook" version="5.0" xml:id="SIAdapterLowerPrefix">
<title>Kafka Adapter</title>
<para>
The Spring Integration Kafka Adapter provides...
</para>
<itemizedlist>
<listitem>
<para><emphasis><link linkend='SIAdapterLowerPrefix-outbound-channel-adapter'>Outbound Channel adapter</link></emphasis></para>
</listitem>
<listitem>
<para><emphasis><link linkend='SIAdapterLowerPrefix-outbound-gateway'>Outbound Gateway</link></emphasis></para>
</listitem>
<listitem>
<para><emphasis><link linkend='SIAdapterLowerPrefix-inbound-channel-adapter'>Inbound Channel Adapter</link></emphasis></para>
</listitem>
</itemizedlist>
<section id="jpa-java-implementation">
<title>Java Implementation</title>
<para>Each of the provided components will use the
<classname>org.springframework.integration.kafka.core.KafkaExecutor</classname>
class...
</para>
</section>
<section id="jpa-common-configuration-attributes">
<title>Common Configuration Attributes</title>
<para>
Certain configuration parameters are shared amongst all Kafka
components and are described below:
</para>
<para><emphasis role="bold">auto-startup</emphasis></para>
<para>
Lifecycle attribute signaling if this component should
be started during Application Context startup.
Defaults to <code>true</code>.
<emphasis>Optional</emphasis>.
</para>
<para><emphasis role="bold">id</emphasis></para>
<para>
Identifies the underlying Spring bean definition, which
is an instance of either <classname>EventDrivenConsumer</classname>
or <classname>PollingConsumer</classname>.
<emphasis>Optional</emphasis>.
</para>
</section>
<section id="SIAdapterLowerPrefix-outbound-channel-adapter">
<title>Outbound Channel Adapter</title>
<para>
The Kafka Outbound channel adapter allows you to...
</para>
</section>
<section id="SIAdapterLowerPrefix-outbound-gateway">
<title>Outbound Gateway</title>
<para>
Outbound gateways are similar to outbound channel adapters except that it can also be used to
get a result on the <emphasis>reply channel</emphasis> after performing
the given...
</para>
</section>
<section id="SIAdapterLowerPrefix-inbound-channel-adapter">
<title>Inbound Channel Adapter</title>
<para>
An inbound channel adapter is used to execute...
</para>
</section>
</chapter>

4
spring-integration-kafka/src/reference/docbook/history.xml Normal file
View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<appendix xmlns="http://docbook.org/ns/docbook" version="5.0" xml:id="history">
<title>Change History</title>
</appendix>

BIN
spring-integration-kafka/src/reference/docbook/images/logo.png Normal file
View File

Binary file not shown.

67
spring-integration-kafka/src/reference/docbook/index.xml Normal file
View File

@@ -0,0 +1,67 @@
<?xml version="1.0" encoding="UTF-8"?>
<book xmlns="http://docbook.org/ns/docbook" version="5.0"
xml:id="spring-integration-reference" xmlns:xi="http://www.w3.org/2001/XInclude">
<bookinfo>
<title>Spring Integration Kafka Adapter</title>
<titleabbrev>Kafka Adapter ${version}</titleabbrev>
<productname>Spring Integration</productname>
<releaseinfo>${version}</releaseinfo>
<!-- TODO: this isn't showing up. -->
<mediaobject>
<imageobject role="fo">
<imagedata fileref="images/logo.png" format="PNG" align="center" />
</imageobject>
<imageobject role="html">
<imagedata fileref="images/logo.png" format="PNG" align="center" />
</imageobject>
</mediaobject>
<!-- END TODO -->
<authorgroup>
<author><firstname>Soby Chacko</firstname></author>
</authorgroup>
<legalnotice>
<para>© SpringSource Inc., 2012</para>
</legalnotice>
</bookinfo>
<toc></toc>
<part id="whats-new-part">
<title>What's new?</title>
<partintro id="spring-integration-intro">
<para>
For those who are already familiar with Spring Integration, this
chapter
provides a brief overview of the new features of version 2.2. If you are
interested in the changes and features, that were introduced in
earlier
versions, please take a look at chapter:
<xref linkend="history" />
</para>
</partintro>
<xi:include href="./whats-new.xml" />
</part>
<part id="spring-integration-adapters">
<title>Integration Adapters</title>
<partintro id="spring-integration-adapters">
<para>This section covers the various Channel Adapters and Messaging
Gateways provided
by Spring Integration to support Message-based communication with
external systems.
</para>
</partintro>
<xi:include href="./SIAdapterLowerPrefix.xml" />
</part>
<part id="spring-integration-appendices">
<title>Appendices</title>
<partintro id="spring-integration-adapters">
<para>Advanced Topics and Additional Resources</para>
</partintro>
<xi:include href="./history.xml" />
</part>
</book>

15
spring-integration-kafka/src/reference/docbook/resources.xml Normal file
View File

@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<appendix xmlns="http://docbook.org/ns/docbook" version="5.0" xml:id="resources">
<title>Additional Resources</title>
<section id="resources-home">
<title>Spring Integration Home</title>
<para>
The definitive source of information about Spring Integration is the
<ulink url="http://www.springsource.org/spring-integration">Spring Integration Home</ulink> at
<ulink url="http://www.springsource.org">http://www.springsource.org</ulink>. That site serves as a hub of
information and is the best place to find up-to-date announcements about the project as well as links to
articles, blogs, and new sample applications.
</para>
</section>
</appendix>

8
spring-integration-kafka/src/reference/docbook/whats-new.xml Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<chapter xmlns="http://docbook.org/ns/docbook" version="5.0" xml:id="whats-new">
<title>What's new?</title>
<para>
This chapter provides an overview of the new features and improvements
that have been added to the Kafka Adapter:
</para>
</chapter>

47
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests.java Normal file
View File

@@ -0,0 +1,47 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import junit.framework.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.integration.kafka.support.ConsumerMetadata;
import org.springframework.integration.kafka.support.KafkaConsumerContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author Soby Chacko
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
public class KafkaConsumerContextParserTests {
@Autowired
private ApplicationContext appContext;
@Test
@SuppressWarnings("unchecked")
public void testConsumerContextConfiguration() {
final KafkaConsumerContext consumerContext = appContext.getBean("consumerContext", KafkaConsumerContext.class);
Assert.assertNotNull(consumerContext);
final ConsumerMetadata cm = appContext.getBean("consumerMetadata_default1", ConsumerMetadata.class);
Assert.assertNotNull(cm);
}
}

45
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests.java Normal file
View File

@@ -0,0 +1,45 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import junit.framework.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.integration.endpoint.SourcePollingChannelAdapter;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
public class KafkaInboundAdapterParserTests {
@Autowired
private ApplicationContext appContext;
/**
* Test method for {@link org.springframework.integration.kafka.config.xml.KafkaInboundChannelAdapterParser#parseSource(org.w3c.dom.Element, org.springframework.beans.factory.xml.ParserContext)}.
*/
@Test
public void testParseSourceElementParserContext() throws Exception {
final SourcePollingChannelAdapter adapter = appContext.getBean("kafkaInboundChannelAdapter",
SourcePollingChannelAdapter.class);
Assert.assertNotNull(adapter);
Assert.assertFalse(adapter.isAutoStartup());
}
}

46
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests.java Normal file
View File

@@ -0,0 +1,46 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import junit.framework.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.integration.endpoint.PollingConsumer;
import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler;
import org.springframework.integration.kafka.support.KafkaProducerContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
public class KafkaOutboundAdapterParserTests {
@Autowired
private ApplicationContext appContext;
@Test
public void testOutboundAdapterConfiguration(){
final PollingConsumer pollingConsumer = appContext.getBean("kafkaOutboundChannelAdapter", PollingConsumer.class);
final KafkaProducerMessageHandler messageHandler = appContext.getBean(KafkaProducerMessageHandler.class);
Assert.assertNotNull(pollingConsumer);
Assert.assertNotNull(messageHandler);
final KafkaProducerContext producerContext = messageHandler.getKafkaProducerContext();
Assert.assertNotNull(producerContext);
Assert.assertEquals(producerContext.getTopicsConfiguration().size(), 2);
}
}

73
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests.java Normal file
View File

@@ -0,0 +1,73 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import junit.framework.Assert;
import kafka.javaapi.producer.Producer;
import kafka.serializer.Encoder;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.integration.kafka.support.KafkaProducerContext;
import org.springframework.integration.kafka.support.ProducerConfiguration;
import org.springframework.integration.kafka.support.ProducerMetadata;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import java.util.Map;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
public class KafkaProducerContextParserTests {
@Autowired
private ApplicationContext appContext;
@Test
@SuppressWarnings("unchecked")
public void testProducerContextConfiguration(){
final KafkaProducerContext producerContext = appContext.getBean("producerContext", KafkaProducerContext.class);
Assert.assertNotNull(producerContext);
final Map<String, ProducerConfiguration> topicConfigurations = producerContext.getTopicsConfiguration();
Assert.assertEquals(topicConfigurations.size(), 2);
final ProducerConfiguration producerConfigurationTest1 = topicConfigurations.get("producerConfiguration_test1");
Assert.assertNotNull(producerConfigurationTest1);
final ProducerMetadata producerMetadataTest1 = producerConfigurationTest1.getProducerMetadata();
Assert.assertEquals(producerMetadataTest1.getTopic(), "test1");
Assert.assertEquals(producerMetadataTest1.getCompressionCodec(), "0");
Assert.assertEquals(producerMetadataTest1.getKeyClassType(), java.lang.String.class);
Assert.assertEquals(producerMetadataTest1.getValueClassType(), java.lang.String.class);
final Encoder valueEncoder = appContext.getBean("valueEncoder", Encoder.class);
Assert.assertEquals(producerMetadataTest1.getValueEncoder(), valueEncoder);
Assert.assertEquals(producerMetadataTest1.getKeyEncoder(), valueEncoder);
final Producer producerTest1 = appContext.getBean("prodFactory_test1", Producer.class);
Assert.assertEquals(producerConfigurationTest1, new ProducerConfiguration(producerMetadataTest1, producerTest1));
final ProducerConfiguration producerConfigurationTest2 = topicConfigurations.get("producerConfiguration_" + "test2");
Assert.assertNotNull(producerConfigurationTest2);
final ProducerMetadata producerMetadataTest2 = producerConfigurationTest2.getProducerMetadata();
Assert.assertEquals(producerMetadataTest2.getTopic(), "test2");
Assert.assertEquals(producerMetadataTest2.getCompressionCodec(), "0");
final Producer producerTest2 = appContext.getBean("prodFactory_test2", Producer.class);
Assert.assertEquals(producerConfigurationTest2, new ProducerConfiguration(producerMetadataTest2, producerTest2));
}
}

59
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests.java Normal file
View File

@@ -0,0 +1,59 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.config.xml;
import junit.framework.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.integration.kafka.core.ZookeeperConnectDefaults;
import org.springframework.integration.kafka.support.ZookeeperConnect;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author Soby Chacko
* @since 1.0
*
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
public class ZookeeperConnectParserTests {
@Autowired
private ApplicationContext appContext;
@Test
public void testCustomKafkaBrokerConfiguration() {
final ZookeeperConnect broker = appContext.getBean("zookeeperConnect", ZookeeperConnect.class);
Assert.assertEquals("localhost:2181", broker.getZkConnect());
Assert.assertEquals("10000", broker.getZkConnectionTimeout());
Assert.assertEquals("10000", broker.getZkSessionTimeout());
Assert.assertEquals("200", broker.getZkSyncTime());
}
@Test
public void testDefaultKafkaBrokerConfiguration() {
final ZookeeperConnect broker = appContext.getBean("defaultZookeeperConnect", ZookeeperConnect.class);
Assert.assertEquals(ZookeeperConnectDefaults.ZK_CONNECT, broker.getZkConnect());
Assert.assertEquals(ZookeeperConnectDefaults.ZK_CONNECTION_TIMEOUT, broker.getZkConnectionTimeout());
Assert.assertEquals(ZookeeperConnectDefaults.ZK_SESSION_TIMEOUT, broker.getZkSessionTimeout());
Assert.assertEquals(ZookeeperConnectDefaults.ZK_SYNC_TIME, broker.getZkSyncTime());
}
}

59
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/serializer/AvroBackedKafkaSerializerTest.java Normal file
View File

@@ -0,0 +1,59 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.serializer;
import junit.framework.Assert;
import org.junit.Test;
import org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaDecoder;
import org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaEncoder;
import org.springframework.integration.kafka.test.utils.TestObject;
/**
* @author Soby Chacko
* @since 1.0
*/
public class AvroBackedKafkaSerializerTest {
@Test
@SuppressWarnings("unchecked")
public void testDecodePlainSchema() {
final AvroBackedKafkaEncoder avroBackedKafkaEncoder = new AvroBackedKafkaEncoder(TestObject.class);
final TestObject testObject = new TestObject();
testObject.setTestData1("\"Test Data1\"");
testObject.setTestData2(1);
final byte[] data = avroBackedKafkaEncoder.toBytes(testObject);
final AvroBackedKafkaDecoder avroBackedKafkaDecoder = new AvroBackedKafkaDecoder(TestObject.class);
final TestObject decodedFbu = (TestObject) avroBackedKafkaDecoder.fromBytes(data);
Assert.assertEquals(testObject.getTestData1(), decodedFbu.getTestData1());
Assert.assertEquals(testObject.getTestData2(), decodedFbu.getTestData2());
}
@Test
@SuppressWarnings("unchecked")
public void anotherTest() {
final AvroBackedKafkaEncoder avroBackedKafkaEncoder = new AvroBackedKafkaEncoder(java.lang.String.class);
final String testString = "Testing Avro";
final byte[] data = avroBackedKafkaEncoder.toBytes(testString);
final AvroBackedKafkaDecoder avroBackedKafkaDecoder = new AvroBackedKafkaDecoder(java.lang.String.class);
final String decodedS = (String) avroBackedKafkaDecoder.fromBytes(data);
Assert.assertEquals(testString, decodedS);
}
}

306
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ConsumerConfigurationTests.java Normal file
View File

@@ -0,0 +1,306 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import junit.framework.Assert;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.message.MessageAndMetadata;
import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author Soby Chacko
*/
public class ConsumerConfigurationTests {
@Test
@SuppressWarnings("unchecked")
public void testReceiveMessageForSingleTopicFromSingleStream() {
final ConsumerMetadata consumerMetadata = Mockito.mock(ConsumerMetadata.class);
final ConsumerConnectionProvider consumerConnectionProvider =
Mockito.mock(ConsumerConnectionProvider.class);
final MessageLeftOverTracker messageLeftOverTracker = Mockito.mock(MessageLeftOverTracker.class);
final ConsumerConnector consumerConnector = Mockito.mock(ConsumerConnector.class);
Mockito.when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector);
final ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(consumerMetadata,
consumerConnectionProvider, messageLeftOverTracker);
consumerConfiguration.setMaxMessages(1);
final KafkaStream stream = Mockito.mock(KafkaStream.class);
final List<KafkaStream<byte[], byte[]>> streams = new ArrayList<KafkaStream<byte[], byte[]>>();
streams.add(stream);
final Map<String, List<KafkaStream<byte[], byte[]>>> messageStreams = new HashMap<String, List<KafkaStream<byte[], byte[]>>>();
messageStreams.put("topic", streams);
Mockito.when(consumerConfiguration.getConsumerMapWithMessageStreams()).thenReturn(messageStreams);
final ConsumerIterator iterator = Mockito.mock(ConsumerIterator.class);
Mockito.when(stream.iterator()).thenReturn(iterator);
final MessageAndMetadata messageAndMetadata = Mockito.mock(MessageAndMetadata.class);
Mockito.when(iterator.next()).thenReturn(messageAndMetadata);
Mockito.when(messageAndMetadata.message()).thenReturn("got message");
Mockito.when(messageAndMetadata.topic()).thenReturn("topic");
Mockito.when(messageAndMetadata.partition()).thenReturn(1);
final Map<String, Map<Integer, List<Object>>> messages = consumerConfiguration.receive();
Assert.assertEquals(messages.size(), 1);
Assert.assertEquals(messages.get("topic").size(), 1);
Assert.assertEquals(messages.get("topic").get(1).get(0), "got message");
Mockito.verify(stream, Mockito.times(1)).iterator();
Mockito.verify(iterator, Mockito.times(1)).next();
Mockito.verify(messageAndMetadata, Mockito.times(1)).message();
Mockito.verify(messageAndMetadata, Mockito.times(1)).topic();
}
@Test
@SuppressWarnings("unchecked")
public void testReceiveMessageForSingleTopicFromMultipleStreams() {
final ConsumerMetadata consumerMetadata = Mockito.mock(ConsumerMetadata.class);
final ConsumerConnectionProvider consumerConnectionProvider =
Mockito.mock(ConsumerConnectionProvider.class);
final MessageLeftOverTracker messageLeftOverTracker = Mockito.mock(MessageLeftOverTracker.class);
final ConsumerConnector consumerConnector = Mockito.mock(ConsumerConnector.class);
Mockito.when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector);
final ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(consumerMetadata,
consumerConnectionProvider, messageLeftOverTracker);
consumerConfiguration.setMaxMessages(3);
final KafkaStream stream1 = Mockito.mock(KafkaStream.class);
final KafkaStream stream2 = Mockito.mock(KafkaStream.class);
final KafkaStream stream3 = Mockito.mock(KafkaStream.class);
final List<KafkaStream<byte[], byte[]>> streams = new ArrayList<KafkaStream<byte[], byte[]>>();
streams.add(stream1);
streams.add(stream2);
streams.add(stream3);
final Map<String, List<KafkaStream<byte[], byte[]>>> messageStreams = new HashMap<String, List<KafkaStream<byte[], byte[]>>>();
messageStreams.put("topic", streams);
Mockito.when(consumerConfiguration.getConsumerMapWithMessageStreams()).thenReturn(messageStreams);
final ConsumerIterator iterator1 = Mockito.mock(ConsumerIterator.class);
final ConsumerIterator iterator2 = Mockito.mock(ConsumerIterator.class);
final ConsumerIterator iterator3 = Mockito.mock(ConsumerIterator.class);
Mockito.when(stream1.iterator()).thenReturn(iterator1);
Mockito.when(stream2.iterator()).thenReturn(iterator2);
Mockito.when(stream3.iterator()).thenReturn(iterator3);
final MessageAndMetadata messageAndMetadata1 = Mockito.mock(MessageAndMetadata.class);
final MessageAndMetadata messageAndMetadata2 = Mockito.mock(MessageAndMetadata.class);
final MessageAndMetadata messageAndMetadata3 = Mockito.mock(MessageAndMetadata.class);
Mockito.when(iterator1.next()).thenReturn(messageAndMetadata1);
Mockito.when(iterator2.next()).thenReturn(messageAndMetadata2);
Mockito.when(iterator3.next()).thenReturn(messageAndMetadata3);
Mockito.when(messageAndMetadata1.message()).thenReturn("got message");
Mockito.when(messageAndMetadata1.topic()).thenReturn("topic");
Mockito.when(messageAndMetadata1.partition()).thenReturn(1);
Mockito.when(messageAndMetadata2.message()).thenReturn("got message");
Mockito.when(messageAndMetadata2.topic()).thenReturn("topic");
Mockito.when(messageAndMetadata2.partition()).thenReturn(2);
Mockito.when(messageAndMetadata3.message()).thenReturn("got message");
Mockito.when(messageAndMetadata3.topic()).thenReturn("topic");
Mockito.when(messageAndMetadata3.partition()).thenReturn(3);
final Map<String, Map<Integer, List<Object>>> messages = consumerConfiguration.receive();
Assert.assertEquals(messages.size(), 1);
int sum = 0;
final Map<Integer, List<Object>> values = messages.get("topic");
for (final List<Object> l : values.values()) {
sum += l.size();
}
Assert.assertEquals(sum, 3);
}
@Test
@SuppressWarnings("unchecked")
public void testReceiveMessageForMultipleTopicsFromMultipleStreams() {
final ConsumerMetadata consumerMetadata = Mockito.mock(ConsumerMetadata.class);
final ConsumerConnectionProvider consumerConnectionProvider =
Mockito.mock(ConsumerConnectionProvider.class);
final MessageLeftOverTracker messageLeftOverTracker = Mockito.mock(MessageLeftOverTracker.class);
final ConsumerConnector consumerConnector = Mockito.mock(ConsumerConnector.class);
Mockito.when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector);
final ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(consumerMetadata,
consumerConnectionProvider, messageLeftOverTracker);
consumerConfiguration.setMaxMessages(9);
final KafkaStream stream1 = Mockito.mock(KafkaStream.class);
final KafkaStream stream2 = Mockito.mock(KafkaStream.class);
final KafkaStream stream3 = Mockito.mock(KafkaStream.class);
final List<KafkaStream<byte[], byte[]>> streams = new ArrayList<KafkaStream<byte[], byte[]>>();
streams.add(stream1);
streams.add(stream2);
streams.add(stream3);
final Map<String, List<KafkaStream<byte[], byte[]>>> messageStreams = new HashMap<String, List<KafkaStream<byte[], byte[]>>>();
messageStreams.put("topic1", streams);
messageStreams.put("topic2", streams);
messageStreams.put("topic3", streams);
Mockito.when(consumerConfiguration.getConsumerMapWithMessageStreams()).thenReturn(messageStreams);
final ConsumerIterator iterator1 = Mockito.mock(ConsumerIterator.class);
final ConsumerIterator iterator2 = Mockito.mock(ConsumerIterator.class);
final ConsumerIterator iterator3 = Mockito.mock(ConsumerIterator.class);
Mockito.when(stream1.iterator()).thenReturn(iterator1);
Mockito.when(stream2.iterator()).thenReturn(iterator2);
Mockito.when(stream3.iterator()).thenReturn(iterator3);
final MessageAndMetadata messageAndMetadata1 = Mockito.mock(MessageAndMetadata.class);
final MessageAndMetadata messageAndMetadata2 = Mockito.mock(MessageAndMetadata.class);
final MessageAndMetadata messageAndMetadata3 = Mockito.mock(MessageAndMetadata.class);
Mockito.when(iterator1.next()).thenReturn(messageAndMetadata1);
Mockito.when(iterator2.next()).thenReturn(messageAndMetadata2);
Mockito.when(iterator3.next()).thenReturn(messageAndMetadata3);
Mockito.when(messageAndMetadata1.message()).thenReturn("got message1");
Mockito.when(messageAndMetadata1.topic()).thenReturn("topic1");
Mockito.when(messageAndMetadata1.partition()).thenAnswer(getAnswer());
Mockito.when(messageAndMetadata2.message()).thenReturn("got message2");
Mockito.when(messageAndMetadata2.topic()).thenReturn("topic2");
Mockito.when(messageAndMetadata1.partition()).thenAnswer(getAnswer());
Mockito.when(messageAndMetadata3.message()).thenReturn("got message3");
Mockito.when(messageAndMetadata3.topic()).thenReturn("topic3");
Mockito.when(messageAndMetadata1.partition()).thenAnswer(getAnswer());
final Map<String, Map<Integer, List<Object>>> messages = consumerConfiguration.receive();
int sum = 0;
final Collection<Map<Integer, List<Object>>> values = messages.values();
for (final Map<Integer, List<Object>> m : values) {
for (final List<Object> l : m.values()) {
sum += l.size();
}
}
Assert.assertEquals(sum, 9);
}
private Answer<Object> getAnswer() {
return new Answer<Object>() {
private int count = 0;
@Override
public Object answer(final InvocationOnMock invocation) throws Throwable {
if (count++ == 1) {
return 1;
} else if (count++ == 2) {
return 2;
}
return 3;
}
};
}
@Test
@SuppressWarnings("unchecked")
public void testReceiveMessageAndVerifyMessageLeftoverFromPreviousPollAreTakenFirst() {
final ConsumerMetadata consumerMetadata = Mockito.mock(ConsumerMetadata.class);
final ConsumerConnectionProvider consumerConnectionProvider =
Mockito.mock(ConsumerConnectionProvider.class);
final MessageLeftOverTracker messageLeftOverTracker = Mockito.mock(MessageLeftOverTracker.class);
final ConsumerConnector consumerConnector = Mockito.mock(ConsumerConnector.class);
Mockito.when(messageLeftOverTracker.getCurrentCount()).thenReturn(3);
final MessageAndMetadata m1 = new MessageAndMetadata("key1", "value1", "topic1", 1, 1L);
final MessageAndMetadata m2 = new MessageAndMetadata("key2", "value2", "topic2", 1, 1L);
final MessageAndMetadata m3 = new MessageAndMetadata("key1", "value3", "topic3", 1, 1L);
final List<MessageAndMetadata> mList = new ArrayList<MessageAndMetadata>();
mList.add(m1);
mList.add(m2);
mList.add(m3);
Mockito.when(messageLeftOverTracker.getMessageLeftOverFromPreviousPoll()).thenReturn(mList);
Mockito.when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector);
final ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(consumerMetadata,
consumerConnectionProvider, messageLeftOverTracker);
consumerConfiguration.setMaxMessages(5);
final KafkaStream stream = Mockito.mock(KafkaStream.class);
final List<KafkaStream<byte[], byte[]>> streams = new ArrayList<KafkaStream<byte[], byte[]>>();
streams.add(stream);
final Map<String, List<KafkaStream<byte[], byte[]>>> messageStreams = new HashMap<String, List<KafkaStream<byte[], byte[]>>>();
messageStreams.put("topic1", streams);
Mockito.when(consumerConfiguration.getConsumerMapWithMessageStreams()).thenReturn(messageStreams);
final ConsumerIterator iterator = Mockito.mock(ConsumerIterator.class);
Mockito.when(stream.iterator()).thenReturn(iterator);
final MessageAndMetadata messageAndMetadata = Mockito.mock(MessageAndMetadata.class);
Mockito.when(iterator.next()).thenReturn(messageAndMetadata);
Mockito.when(messageAndMetadata.message()).thenReturn("got message");
Mockito.when(messageAndMetadata.topic()).thenReturn("topic1");
Mockito.when(messageAndMetadata.partition()).thenReturn(1);
final Map<String, Map<Integer, List<Object>>> messages = consumerConfiguration.receive();
int sum = 0;
final Collection<Map<Integer, List<Object>>> values = messages.values();
for (final Map<Integer, List<Object>> m : values) {
for (final List<Object> l : m.values()) {
sum += l.size();
}
}
Assert.assertEquals(sum, 5);
Assert.assertTrue(messages.containsKey("topic1"));
Assert.assertTrue(messages.containsKey("topic2"));
Assert.assertTrue(messages.containsKey("topic3"));
Assert.assertTrue(valueFound(messages.get("topic1").get(1), "value1"));
Assert.assertTrue(valueFound(messages.get("topic2").get(1), "value2"));
Assert.assertTrue(valueFound(messages.get("topic3").get(1), "value3"));
}
private boolean valueFound(final List<Object> l, final String value){
for (final Object o : l){
if (value.equals(o)){
return true;
}
}
return false;
}
}

80
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaConsumerContextTest.java Normal file
View File

@@ -0,0 +1,80 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.ListableBeanFactory;
import org.springframework.integration.Message;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author Soby Chacko
*/
public class KafkaConsumerContextTest {
@Test
public void testMergeResultsFromMultipleConsumerConfiguration() {
final KafkaConsumerContext kafkaConsumerContext = new KafkaConsumerContext();
final ListableBeanFactory beanFactory = Mockito.mock(ListableBeanFactory.class);
final ConsumerConfiguration consumerConfiguration1 = Mockito.mock(ConsumerConfiguration.class);
final ConsumerConfiguration consumerConfiguration2 = Mockito.mock(ConsumerConfiguration.class);
final Map<String, ConsumerConfiguration> map = new HashMap<String, ConsumerConfiguration>();
map.put("config1", consumerConfiguration1);
map.put("config2", consumerConfiguration2);
Mockito.when(beanFactory.getBeansOfType(ConsumerConfiguration.class)).thenReturn(map);
kafkaConsumerContext.setBeanFactory(beanFactory);
final Map<String, Map<Integer, List<Object>>> result1 = new HashMap<String, Map<Integer, List<Object>>>();
final List<Object> l1 = new ArrayList<Object>();
l1.add("got message1 - l1");
l1.add("got message2 - l1");
final Map<Integer, List<Object>> innerMap1 = new HashMap<Integer, List<Object>>();
innerMap1.put(1, l1);
result1.put("topic1", innerMap1);
final Map<String, Map<Integer, List<Object>>> result2 = new HashMap<String, Map<Integer, List<Object>>>();
final List<Object> l2 = new ArrayList<Object>();
l2.add("got message1 - l2");
l2.add("got message2 - l2");
l2.add("got message3 - l2");
final Map<Integer, List<Object>> innerMap2 = new HashMap<Integer, List<Object>>();
innerMap2.put(1, l2);
result1.put("topic2", innerMap2);
Mockito.when(consumerConfiguration1.receive()).thenReturn(result1);
Mockito.when(consumerConfiguration2.receive()).thenReturn(result2);
final Message<Map<String, Map<Integer, List<Object>>>> messages = kafkaConsumerContext.receive();
Assert.assertEquals(messages.getPayload().size(), 2);
Assert.assertEquals(messages.getPayload().get("topic1").size(), 1);
Assert.assertEquals(messages.getPayload().get("topic1").get(1).get(0), "got message1 - l1");
Assert.assertEquals(messages.getPayload().get("topic1").get(1).get(1), "got message2 - l1");
Assert.assertEquals(messages.getPayload().get("topic2").size(), 1);
Assert.assertEquals(messages.getPayload().get("topic2").get(1).get(0), "got message1 - l2");
Assert.assertEquals(messages.getPayload().get("topic2").get(1).get(1), "got message2 - l2");
Assert.assertEquals(messages.getPayload().get("topic2").get(1).get(2), "got message3 - l2");
}
}

303
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerConfigurationTests.java Normal file
View File

@@ -0,0 +1,303 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.serializer.DefaultEncoder;
import kafka.serializer.StringEncoder;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.springframework.integration.Message;
import org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaEncoder;
import org.springframework.integration.kafka.test.utils.NonSerializableTestKey;
import org.springframework.integration.kafka.test.utils.NonSerializableTestPayload;
import org.springframework.integration.kafka.test.utils.TestKey;
import org.springframework.integration.kafka.test.utils.TestPayload;
import org.springframework.integration.support.MessageBuilder;
import java.io.ByteArrayInputStream;
import java.io.NotSerializableException;
import java.io.ObjectInputStream;
/**
* @author Soby Chacko
*/
public class ProducerConfigurationTests {
@Test
@SuppressWarnings("unchecked")
public void testSendMessageWithNonDefaultKeyAndValueEncoders() throws Exception {
final ProducerMetadata<String, String> producerMetadata = new ProducerMetadata<String, String>("test");
producerMetadata.setValueEncoder(new StringEncoder(null));
producerMetadata.setKeyEncoder(new StringEncoder(null));
producerMetadata.setKeyClassType(String.class);
producerMetadata.setValueClassType(String.class);
final Producer<String, String> producer = Mockito.mock(Producer.class);
final ProducerConfiguration<String, String> configuration = new ProducerConfiguration<String, String>(producerMetadata, producer);
final Message<String> message = MessageBuilder.withPayload("test message").
setHeader("messageKey", "key")
.setHeader("topic", "test").build();
configuration.send(message);
Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(KeyedMessage.class));
final ArgumentCaptor<KeyedMessage> argument = ArgumentCaptor.forClass(KeyedMessage.class);
Mockito.verify(producer).send(argument.capture());
final KeyedMessage capturedKeyMessage = argument.getValue();
Assert.assertEquals(capturedKeyMessage.key(), "key");
Assert.assertEquals(capturedKeyMessage.message(), "test message");
Assert.assertEquals(capturedKeyMessage.topic(), "test");
}
/**
* User does not set an explicit key/value encoder, but send a serializable object for both key/value
*/
@Test
@SuppressWarnings("unchecked")
public void testSendMessageWithDefaultKeyAndValueEncodersAndCustomSerializableKeyAndPayloadObject() throws Exception {
final ProducerMetadata<byte[], byte[]> producerMetadata = new ProducerMetadata<byte[], byte[]>("test");
producerMetadata.setValueEncoder(new DefaultEncoder(null));
producerMetadata.setKeyEncoder(new DefaultEncoder(null));
final Producer<byte[], byte[]> producer = Mockito.mock(Producer.class);
final ProducerConfiguration<byte[], byte[]> configuration = new ProducerConfiguration<byte[], byte[]>(producerMetadata, producer);
final Message<TestPayload> message = MessageBuilder.withPayload(new TestPayload("part1", "part2")).
setHeader("messageKey", new TestKey("compositePart1", "compositePart2"))
.setHeader("topic", "test").build();
configuration.send(message);
Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(KeyedMessage.class));
final ArgumentCaptor<KeyedMessage> argument = ArgumentCaptor.forClass(KeyedMessage.class);
Mockito.verify(producer).send(argument.capture());
final KeyedMessage capturedKeyMessage = argument.getValue();
final byte[] keyBytes = (byte[])capturedKeyMessage.key();
final ByteArrayInputStream keyInputStream = new ByteArrayInputStream (keyBytes);
final ObjectInputStream keyObjectInputStream = new ObjectInputStream (keyInputStream);
final Object keyObj = keyObjectInputStream.readObject();
final TestKey tk = (TestKey)keyObj;
Assert.assertEquals(tk.getKeyPart1(), "compositePart1");
Assert.assertEquals(tk.getKeyPart2(), "compositePart2");
final byte[] messageBytes = (byte[])capturedKeyMessage.message();
final ByteArrayInputStream messageInputStream = new ByteArrayInputStream (messageBytes);
final ObjectInputStream messageObjectInputStream = new ObjectInputStream (messageInputStream);
final Object messageObj = messageObjectInputStream.readObject();
final TestPayload tp = (TestPayload)messageObj;
Assert.assertEquals(tp.getPart1(), "part1");
Assert.assertEquals(tp.getPart2(), "part2");
Assert.assertEquals(capturedKeyMessage.topic(), "test");
}
/**
* User does not set an explicit key encoder, but a value encoder, and sends the corresponding data
*/
@Test
@SuppressWarnings("unchecked")
public void testSendMessageWithDefaultKeyEncoderAndNonDefaultValueEncoderAndCorrespondingData() throws Exception {
final ProducerMetadata<byte[], TestPayload> producerMetadata = new ProducerMetadata<byte[], TestPayload>("test");
final AvroBackedKafkaEncoder<TestPayload> encoder = new AvroBackedKafkaEncoder<TestPayload>(TestPayload.class);
producerMetadata.setValueEncoder(encoder);
producerMetadata.setKeyEncoder(new DefaultEncoder(null));
producerMetadata.setValueClassType(TestPayload.class);
final Producer<byte[], TestPayload> producer = Mockito.mock(Producer.class);
final ProducerConfiguration<byte[], TestPayload> configuration = new ProducerConfiguration<byte[], TestPayload>(producerMetadata, producer);
final TestPayload tp = new TestPayload("part1", "part2");
final Message<TestPayload> message = MessageBuilder.withPayload(tp).
setHeader("messageKey", "key")
.setHeader("topic", "test").build();
configuration.send(message);
Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(KeyedMessage.class));
final ArgumentCaptor<KeyedMessage> argument = ArgumentCaptor.forClass(KeyedMessage.class);
Mockito.verify(producer).send(argument.capture());
final KeyedMessage capturedKeyMessage = argument.getValue();
final byte[] keyBytes = (byte[])capturedKeyMessage.key();
final ByteArrayInputStream keyInputStream = new ByteArrayInputStream (keyBytes);
final ObjectInputStream keyObjectInputStream = new ObjectInputStream (keyInputStream);
final Object keyObj = keyObjectInputStream.readObject();
Assert.assertEquals("key", keyObj);
Assert.assertEquals(capturedKeyMessage.message(), tp);
Assert.assertEquals(capturedKeyMessage.topic(), "test");
}
/**
* User does set an explicit key encoder, but not a value encoder, and sends the corresponding data
*/
@Test
@SuppressWarnings("unchecked")
public void testSendMessageWithNonDefaultKeyEncoderAndDefaultValueEncoderAndCorrespondingData() throws Exception {
final ProducerMetadata<TestKey, byte[]> producerMetadata = new ProducerMetadata<TestKey, byte[]>("test");
final AvroBackedKafkaEncoder<TestKey> encoder = new AvroBackedKafkaEncoder<TestKey>(TestKey.class);
producerMetadata.setKeyEncoder(encoder);
producerMetadata.setValueEncoder(new DefaultEncoder(null));
producerMetadata.setKeyClassType(TestKey.class);
final Producer<TestKey, byte[]> producer = Mockito.mock(Producer.class);
final ProducerConfiguration<TestKey, byte[]> configuration = new ProducerConfiguration<TestKey, byte[]>(producerMetadata, producer);
final TestKey tk = new TestKey("part1", "part2");
final Message<String> message = MessageBuilder.withPayload("test message").
setHeader("messageKey", tk)
.setHeader("topic", "test").build();
configuration.send(message);
Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(KeyedMessage.class));
final ArgumentCaptor<KeyedMessage> argument = ArgumentCaptor.forClass(KeyedMessage.class);
Mockito.verify(producer).send(argument.capture());
final KeyedMessage capturedKeyMessage = argument.getValue();
Assert.assertEquals(capturedKeyMessage.key(), tk);
final byte[] payloadBytes = (byte[])capturedKeyMessage.message();
final ByteArrayInputStream payloadBis = new ByteArrayInputStream (payloadBytes);
final ObjectInputStream payloadOis = new ObjectInputStream (payloadBis);
final Object payloadObj = payloadOis.readObject();
Assert.assertEquals("test message", payloadObj);
Assert.assertEquals(capturedKeyMessage.topic(), "test");
}
/**
* User does not set an explicit key/value encoder, but send a serializable String key/value pair
*/
@Test
@SuppressWarnings("unchecked")
public void testSendMessageWithDefaultKeyAndValueEncodersAndStringKeyAndValue() throws Exception {
final ProducerMetadata<byte[], byte[]> producerMetadata = new ProducerMetadata<byte[], byte[]>("test");
producerMetadata.setValueEncoder(new DefaultEncoder(null));
producerMetadata.setKeyEncoder(new DefaultEncoder(null));
final Producer<byte[], byte[]> producer = Mockito.mock(Producer.class);
final ProducerConfiguration<byte[], byte[]> configuration = new ProducerConfiguration<byte[], byte[]>(producerMetadata, producer);
final Message<String> message = MessageBuilder.withPayload("test message").
setHeader("messageKey", "key")
.setHeader("topic", "test").build();
configuration.send(message);
Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(KeyedMessage.class));
final ArgumentCaptor<KeyedMessage> argument = ArgumentCaptor.forClass(KeyedMessage.class);
Mockito.verify(producer).send(argument.capture());
final KeyedMessage capturedKeyMessage = argument.getValue();
final byte[] keyBytes = (byte[])capturedKeyMessage.key();
final ByteArrayInputStream keyBis = new ByteArrayInputStream (keyBytes);
final ObjectInputStream keyOis = new ObjectInputStream (keyBis);
final Object keyObj = keyOis.readObject();
Assert.assertEquals("key", keyObj);
final byte[] payloadBytes = (byte[])capturedKeyMessage.message();
final ByteArrayInputStream payloadBis = new ByteArrayInputStream (payloadBytes);
final ObjectInputStream payloadOis = new ObjectInputStream (payloadBis);
final Object payloadObj = payloadOis.readObject();
Assert.assertEquals("test message", payloadObj);
Assert.assertEquals(capturedKeyMessage.topic(), "test");
}
/**
* User does not set an explicit key/value encoder, but send non-serializable object for both key/value
*/
@Test(expected = NotSerializableException.class)
@SuppressWarnings("unchecked")
public void testSendMessageWithDefaultKeyAndValueEncodersButNonSerializableKeyAndValue() throws Exception {
final ProducerMetadata<byte[], byte[]> producerMetadata = new ProducerMetadata<byte[], byte[]>("test");
producerMetadata.setValueEncoder(new DefaultEncoder(null));
producerMetadata.setKeyEncoder(new DefaultEncoder(null));
final Producer<byte[], byte[]> producer = Mockito.mock(Producer.class);
final ProducerConfiguration<byte[], byte[]> configuration = new ProducerConfiguration<byte[], byte[]>(producerMetadata, producer);
final Message<NonSerializableTestPayload> message = MessageBuilder.withPayload(new NonSerializableTestPayload("part1", "part2")).
setHeader("messageKey", new NonSerializableTestKey("compositePart1", "compositePart2"))
.setHeader("topic", "test").build();
configuration.send(message);
}
/**
* User does not set an explicit key/value encoder, but send non-serializable key and serializable value
*/
@Test(expected = NotSerializableException.class)
@SuppressWarnings("unchecked")
public void testSendMessageWithDefaultKeyAndValueEncodersButNonSerializableKeyAndSerializableValue() throws Exception {
final ProducerMetadata<byte[], byte[]> producerMetadata = new ProducerMetadata<byte[], byte[]>("test");
producerMetadata.setValueEncoder(new DefaultEncoder(null));
producerMetadata.setKeyEncoder(new DefaultEncoder(null));
final Producer<byte[], byte[]> producer = Mockito.mock(Producer.class);
final ProducerConfiguration<byte[], byte[]> configuration = new ProducerConfiguration<byte[], byte[]>(producerMetadata, producer);
final Message<TestPayload> message = MessageBuilder.withPayload(new TestPayload("part1", "part2")).
setHeader("messageKey", new NonSerializableTestKey("compositePart1", "compositePart2"))
.setHeader("topic", "test").build();
configuration.send(message);
}
/**
* User does not set an explicit key/value encoder, but send serializable key and non-serializable value
*/
@Test(expected = NotSerializableException.class)
@SuppressWarnings("unchecked")
public void testSendMessageWithDefaultKeyAndValueEncodersButSerializableKeyAndNonSerializableValue() throws Exception {
final ProducerMetadata<byte[], byte[]> producerMetadata = new ProducerMetadata<byte[], byte[]>("test");
producerMetadata.setValueEncoder(new DefaultEncoder(null));
producerMetadata.setKeyEncoder(new DefaultEncoder(null));
final Producer<byte[], byte[]> producer = Mockito.mock(Producer.class);
final ProducerConfiguration<byte[], byte[]> configuration = new ProducerConfiguration<byte[], byte[]>(producerMetadata, producer);
final Message<NonSerializableTestPayload> message = MessageBuilder.withPayload(new NonSerializableTestPayload("part1", "part2")).
setHeader("messageKey", new TestKey("compositePart1", "compositePart2"))
.setHeader("topic", "test").build();
configuration.send(message);
}
}

63
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerFactoryBeanTests.java Normal file
View File

@@ -0,0 +1,63 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.support;
import junit.framework.Assert;
import kafka.javaapi.producer.Producer;
import org.junit.Test;
import org.mockito.Mockito;
/**
* @author Soby Chacko
*/
public class ProducerFactoryBeanTests {
@Test
public void createProducerWithDefaultMetadata() throws Exception {
final ProducerMetadata<byte[], byte[]> producerMetadata = new ProducerMetadata<byte[], byte[]>("test");
final ProducerMetadata<byte[], byte[]> tm = Mockito.spy(producerMetadata);
final ProducerFactoryBean<byte[], byte[]> producerFactoryBean = new ProducerFactoryBean<byte[], byte[]>(tm, "localhost:9092");
final Producer producer = producerFactoryBean.getObject();
Assert.assertTrue(producer != null);
Mockito.verify(tm, Mockito.times(1)).getPartitioner();
Mockito.verify(tm, Mockito.times(1)).getCompressionCodec();
Mockito.verify(tm, Mockito.times(1)).getValueEncoder();
Mockito.verify(tm, Mockito.times(1)).getKeyEncoder();
Mockito.verify(tm, Mockito.times(1)).isAsync();
Mockito.verify(tm, Mockito.times(0)).getBatchNumMessages();
}
@Test
public void createProducerWithAsyncFeatures() throws Exception {
final ProducerMetadata<byte[], byte[]> producerMetadata = new ProducerMetadata<byte[], byte[]>("test");
producerMetadata.setAsync(true);
producerMetadata.setBatchNumMessages("300");
final ProducerMetadata<byte[], byte[]> tm = Mockito.spy(producerMetadata);
final ProducerFactoryBean<byte[], byte[]> producerFactoryBean = new ProducerFactoryBean<byte[], byte[]>(tm, "localhost:9092");
final Producer producer = producerFactoryBean.getObject();
Assert.assertTrue(producer != null);
Mockito.verify(tm, Mockito.times(1)).getPartitioner();
Mockito.verify(tm, Mockito.times(1)).getCompressionCodec();
Mockito.verify(tm, Mockito.times(1)).getValueEncoder();
Mockito.verify(tm, Mockito.times(1)).getKeyEncoder();
Mockito.verify(tm, Mockito.times(1)).isAsync();
Mockito.verify(tm, Mockito.times(2)).getBatchNumMessages();
}
}

35
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/NonSerializableTestKey.java Normal file
View File

@@ -0,0 +1,35 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.test.utils;
public class NonSerializableTestKey {
private final String keyPart1;
private final String keyPart2;
public NonSerializableTestKey(final String keyPart1, final String keyPart2) {
this.keyPart1 = keyPart1;
this.keyPart2 = keyPart2;
}
public String getKeyPart1() {
return keyPart1;
}
public String getKeyPart2() {
return keyPart2;
}
}

34
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/NonSerializableTestPayload.java Normal file
View File

@@ -0,0 +1,34 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.test.utils;
public class NonSerializableTestPayload {
private final String part1;
private final String part2;
public NonSerializableTestPayload(final String part1, final String part2) {
this.part1 = part1;
this.part2 = part2;
}
public String getPart1() {
return part1;
}
public String getPart2() {
return part2;
}
}

38
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestKey.java Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.test.utils;
import java.io.Serializable;
public class TestKey implements Serializable {
private static final long serialVersionUID = -6415387283545560656L;
private final String keyPart1;
private final String keyPart2;
public TestKey(final String keyPart1, final String keyPart2) {
this.keyPart1 = keyPart1;
this.keyPart2 = keyPart2;
}
public String getKeyPart1() {
return keyPart1;
}
public String getKeyPart2() {
return keyPart2;
}
}

38
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestObject.java Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.test.utils;
public class TestObject {
public String testData1;
public int testData2;
public String getTestData1() {
return testData1;
}
public void setTestData1(final String testData1) {
this.testData1 = testData1;
}
public int getTestData2() {
return testData2;
}
public void setTestData2(final int testData2) {
this.testData2 = testData2;
}
}

38
spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestPayload.java Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kafka.test.utils;
import java.io.Serializable;
public class TestPayload implements Serializable {
private static final long serialVersionUID = -8560378224929007403L;
private final String part1;
private final String part2;
public TestPayload(final String part1, final String part2){
this.part1 = part1;
this.part2 = part2;
}
public String getPart1() {
return part1;
}
public String getPart2() {
return part2;
}
}

8
spring-integration-kafka/src/test/resources/log4j.properties Normal file
View File

@@ -0,0 +1,8 @@
log4j.rootCategory=WARN, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{HH:mm:ss.SSS} %-5p [%t][%c] %m%n
log4j.category.org.springframework.integration=WARN
log4j.category.org.springframework.integration.kafka=INFO

30
spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaConsumerContextParserTests-context.xml Normal file
View File

@@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<int-kafka:zookeeper-connect id="zookeeperConnect" zk-connect="localhost:2181" zk-connection-timeout="6000"
zk-session-timeout="6000"
zk-sync-time="2000"/>
<bean id="valueDecoder" class="org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaDecoder">
<constructor-arg type="java.lang.Class" value="java.lang.String"/>
</bean>
<int-kafka:consumer-context id="consumerContext"
consumer-timeout="4000"
zookeeper-connect="zookeeperConnect">
<int-kafka:consumer-configurations>
<int-kafka:consumer-configuration group-id="default1"
value-decoder="valueDecoder"
key-decoder="valueDecoder"
max-messages="5000">
<int-kafka:topic id="test1" streams="4"/>
<int-kafka:topic id="test2" streams="4"/>
</int-kafka:consumer-configuration>
</int-kafka:consumer-configurations>
</int-kafka:consumer-context>
</beans>

17
spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterCommon-context.xml Normal file
View File

@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:stream="http://www.springframework.org/schema/integration/stream"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/integration/stream http://www.springframework.org/schema/integration/stream/spring-integration-stream.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">
<context:component-scan base-package="org.springframework.integration.kafka.inbound"></context:component-scan>
<int:channel id="inputFromKafka">
</int:channel>
<stream:stdout-channel-adapter id="stdout" channel="inputFromKafka" append-newline="true"/>
</beans>

40
spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterParserTests-context.xml Normal file
View File

@@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<import resource="kafkaInboundAdapterCommon-context.xml"/>
<int-kafka:zookeeper-connect id="zookeeperConnect" zk-connect="localhost:2181" zk-connection-timeout="6000"
zk-session-timeout="6000"
zk-sync-time="2000" />
<int-kafka:inbound-channel-adapter id="kafkaInboundChannelAdapter"
kafka-consumer-context-ref="consumerContext"
auto-startup="false"
channel="inputFromKafka">
<int:poller fixed-delay="10" time-unit="MILLISECONDS" max-messages-per-poll="5"/>
</int-kafka:inbound-channel-adapter>
<bean id="valueDecoder" class="org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaDecoder">
<constructor-arg type="java.lang.Class" value="java.lang.String" />
</bean>
<int-kafka:consumer-context id="consumerContext"
consumer-timeout="4000"
zookeeper-connect="zookeeperConnect">
<int-kafka:consumer-configurations>
<int-kafka:consumer-configuration group-id="default"
value-decoder="valueDecoder"
key-decoder="valueDecoder"
max-messages="5000">
<int-kafka:topic id="test1" streams="4"/>
<int-kafka:topic id="test2" streams="4"/>
</int-kafka:consumer-configuration>
</int-kafka:consumer-configurations>
</int-kafka:consumer-context>
</beans>

44
spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaOutboundAdapterParserTests-context.xml Normal file
View File

@@ -0,0 +1,44 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xmlns:task="http://www.springframework.org/schema/task"
xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task.xsd">
<int:channel id="inputToKafka">
<int:queue/>
</int:channel>
<int-kafka:outbound-channel-adapter id="kafkaOutboundChannelAdapter"
kafka-producer-context-ref="kafkaProducerContext"
auto-startup="false"
channel="inputToKafka"
>
<int:poller fixed-delay="1000" time-unit="MILLISECONDS" receive-timeout="0" task-executor="taskExecutor"/>
</int-kafka:outbound-channel-adapter>
<task:executor id="taskExecutor" pool-size="5" keep-alive="120" queue-capacity="500"/>
<bean id="kafkaEncoder" class="org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaEncoder">
<constructor-arg value="java.lang.String" />
</bean>
<int-kafka:producer-context id="kafkaProducerContext">
<int-kafka:producer-configurations>
<int-kafka:producer-configuration broker-list="localhost:9092"
key-class-type="java.lang.String"
value-class-type="java.lang.String"
topic="test1"
value-encoder="kafkaEncoder"
key-encoder="kafkaEncoder"
compression-codec="default"/>
<int-kafka:producer-configuration broker-list="localhost:9092"
topic="test2"
compression-codec="default"/>
</int-kafka:producer-configurations>
</int-kafka:producer-context>
</beans>

26
spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaProducerContextParserTests-context.xml Normal file
View File

@@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<int-kafka:producer-context id="producerContext">
<int-kafka:producer-configurations>
<int-kafka:producer-configuration broker-list="localhost:9092"
key-class-type="java.lang.String"
value-class-type="java.lang.String"
key-encoder="valueEncoder"
value-encoder="valueEncoder"
topic="test1"
compression-codec="default"/>
<int-kafka:producer-configuration broker-list="localhost:9092"
topic="test2"
compression-codec="default"/>
</int-kafka:producer-configurations>
</int-kafka:producer-context>
<bean id="valueEncoder" class="org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaEncoder">
<constructor-arg value="java.lang.String" />
</bean>
</beans>

14
spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/zookeeperConnectParserTests-context.xml Normal file
View File

@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xsi:schemaLocation="
http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<int-kafka:zookeeper-connect id="zookeeperConnect" zk-connect="localhost:2181" zk-connection-timeout="10000"
zk-session-timeout="10000"
zk-sync-time="200" />
<int-kafka:zookeeper-connect id="defaultZookeeperConnect" />
</beans>