From 6acf985cc1dfc4422f770c950f5cb5d3d388fada Mon Sep 17 00:00:00 2001 From: Marius Bogoevici Date: Fri, 16 Jan 2015 19:04:58 +0200 Subject: [PATCH] INTEXT-123-124-125 Introducing SimpleConsumer API JIRA: https://jira.spring.io/browse/INTEXT-123 https://jira.spring.io/browse/INTEXT-124 https://jira.spring.io/browse/INTEXT-125 The implementation has the following components: - Configuration - allows retrieving a seed broker set either from a preconfigured list or from Zookeeper - ConnectionFactory - manages and caches connections to a set of brokers - Connection - handles low-level SimpleConsumer API calls and converts them to internal objects such as KafkaMessage and KafkaMessageBus - KafkaTemplate - implements higher-level read operations on a Kafka broker - OffsetManager - stores (potentially in a persistent fashion) offsets for a group of consumers, configures/resets initial offsets as necessary - can be configured to start at an arbitrary offset in a partition (e.g. replay since offset 100), or relative to a given timestamp (e.g. replay since Monday) - KafkaMessageListenerContainer - retrieves messages from a given broker for an arbitrary set of partitions or topics (i.e. all partitions in the topics), invoking a MessageListener. Concurrency is adjustable, and allows processing multiple sets of partitions in parallel (while preserving ordering within a partition). Can poll multiple brokers (each on a parallel thread) - KafkaInboundChannelAdapter - channel adapter using KMLC as underlying implementation - AbstractDecodingMessageListener - utility base class for a MessageListener implementation that decodes the payload and key Tests: - Single and multi-broker configurations - Partition subset retrieval - Replicated sets - Compression - Starting offsets (with reset on wrong offset) - Error handling (servers dropping and leader changes) TO DO: - namespace support Remove warnings Cleanup Use ZookeeperConnect for configuring Zookeeper Fixes, including: - Extracted Connection and ConnectionFactory interfaces - Made fields final where necessary - Fixed Javadoc - Added assertions - Corrected headers Updated copyright to 2015 Make connectivity parameters: timeouts, buffer size, minimum fetch size, fetch timeout configurable via ConnectionFactory. Remove sleep in KafkaMessageListenerContainer Add javadocs. Addressing PR comments: - renamed KafkaInboundChannelAdapter to KafkaMessageDrivenChannelAdapter - removed unnecessary setters - close ZkClient properly - fix version Must use TopicAndPartition as key Log messages too via LoggingErrorHandler Newline - Make `MessageStoreOffsetManager` implement `Closeable` and `Flushable` - Fix the configuration of `ErrorHandler` We don't really need this initialization - it's premature optimization Use ':' instead of ' ' as separator for the key Fixes: - Use 'getPhase' from superclass - more cleanup Defer to the superclass MessageBuilder factory and keep SI_FATAL_WHEN_NO_BEANFACTORY happy Fix warnings in tests Changes to support testing against external brokers - refactored the rules so that an abstraction can be used - topic name is configurable - added (Ignored) TestSingleBrokerExternal that tests against an external broker - refactored OutboundTest to use the new consumer Properly catch TopicExistsException Do not hold state for MetadataStoreOffsetManager In KafkaMessageListenerContainer FetcherTasks will block if they have no partitions to listen to (to do - use separate monitors per task) Updated test for offset reset to latest to wait until an actual refresh event takes place, thus preventing the test from not completing in case of a race condition KafkaMessageDrivenChannelAdapter and KafkaMessageListenerContainer now throw new TopicNotFoundException if the topic does not exist on the broker Renamed test classes to use the Test suffix. Cleanup Test if the component is still running when exiting the wait block Serialization warnings Addressing the last round of PR comments Test configuration cleanup The final polishing: code style, JavaDocs, renaming for test classes to the finish with `*Tests` suffix --- .../kafka/core/AbstractConfiguration.java | 72 +++ .../integration/kafka/core/BrokerAddress.java | 102 ++++ .../core/BrokerAddressListConfiguration.java | 40 ++ .../integration/kafka/core/Configuration.java | 47 ++ .../integration/kafka/core/Connection.java | 91 +++ .../kafka/core/ConnectionFactory.java | 67 ++ .../kafka/core/ConsumerException.java | 41 ++ .../kafka/core/DefaultConnection.java | 322 ++++++++++ .../kafka/core/DefaultConnectionFactory.java | 260 ++++++++ .../integration/kafka/core/FetchRequest.java | 62 ++ .../kafka/core/KafkaConsumerDefaults.java | 41 +- .../integration/kafka/core/KafkaMessage.java | 53 ++ .../kafka/core/KafkaMessageBatch.java | 64 ++ .../kafka/core/KafkaMessageMetadata.java | 65 ++ .../kafka/core/KafkaOperations.java | 31 + .../integration/kafka/core/KafkaTemplate.java | 65 ++ .../integration/kafka/core/Partition.java | 81 +++ .../kafka/core/PartitionBrokerMap.java | 66 ++ .../integration/kafka/core/Result.java | 64 ++ .../integration/kafka/core/ResultBuilder.java | 74 +++ .../kafka/core/TopicNotFoundException.java | 29 + .../kafka/core/ZookeeperConfiguration.java | 99 +++ .../KafkaMessageDrivenChannelAdapter.java | 112 ++++ .../AbstractDecodingMessageListener.java | 62 ++ .../ConcurrentMessageListenerDispatcher.java | 157 +++++ .../kafka/listener/ErrorHandler.java | 31 + .../KafkaMessageListenerContainer.java | 578 ++++++++++++++++++ .../kafka/listener/LoggingErrorHandler.java | 38 ++ .../kafka/listener/MessageListener.java | 35 ++ .../listener/MetadataStoreOffsetManager.java | 188 ++++++ .../kafka/listener/OffsetManager.java | 54 ++ .../QueueingMessageListenerInvoker.java | 126 ++++ .../kafka/listener/package-info.java | 4 + .../kafka/support/KafkaHeaders.java | 4 + .../integration/kafka/util/MessageUtils.java | 43 ++ .../kafka/listener/AbstractBrokerTests.java | 157 +++++ ...AbstractMessageListenerContainerTests.java | 224 +++++++ ...apterWithXmlConfigurationTests-context.xml | 60 ++ ...annelAdapterWithXmlConfigurationTests.java | 71 +++ .../ChannelSendingMessageListener.java | 61 ++ .../DefaultConnectionFactoryTests.java | 87 +++ .../listener/DefaultConnectionTests.java | 126 ++++ ...KafkaMessageDrivenChannelAdapterTests.java | 126 ++++ ...nChannelAdapterWithSpecialOffsetTests.java | 172 ++++++ ...venChannelAdapterWithWrongOffsetTests.java | 319 ++++++++++ .../kafka/listener/MultiBroker20Tests.java | 60 ++ .../kafka/listener/MultiBroker2Tests.java | 103 ++++ .../listener/MultiBroker5ReplicatedTests.java | 102 ++++ ...MultiBroker5ReplicatedWithBounceTests.java | 114 ++++ .../kafka/listener/MultiBroker5Tests.java | 102 ++++ .../listener/SingleBrokerExternalTests.java | 131 ++++ .../kafka/listener/SingleBrokerTests.java | 98 +++ .../SingleBrokerWithCompressionTests.java | 98 +++ .../SingleBrokerWithPartitionSubsetTests.java | 103 ++++ .../kafka/listener/TestPartitioner.java | 50 ++ .../kafka/listener/WrongTopicNameTests.java | 90 +++ .../listener/ZookeeperConfigurationTests.java | 107 ++++ .../kafka/outbound/OutboundTests.java | 109 ++-- .../integration/kafka/rule/KafkaEmbedded.java | 177 ++++++ .../integration/kafka/rule/KafkaRule.java | 44 ++ .../integration/kafka/rule/KafkaRunning.java | 57 +- 61 files changed, 6150 insertions(+), 66 deletions(-) create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/AbstractConfiguration.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddress.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddressListConfiguration.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Configuration.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Connection.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConnectionFactory.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConsumerException.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnection.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnectionFactory.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/FetchRequest.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessage.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageBatch.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageMetadata.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaOperations.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaTemplate.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Partition.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/PartitionBrokerMap.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Result.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ResultBuilder.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/TopicNotFoundException.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConfiguration.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaMessageDrivenChannelAdapter.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingMessageListener.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ConcurrentMessageListenerDispatcher.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ErrorHandler.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaMessageListenerContainer.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LoggingErrorHandler.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MessageListener.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MetadataStoreOffsetManager.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/OffsetManager.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/QueueingMessageListenerInvoker.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/package-info.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/MessageUtils.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractBrokerTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractMessageListenerContainerTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests-context.xml create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelSendingMessageListener.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionFactoryTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithWrongOffsetTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker20Tests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker2Tests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedWithBounceTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5Tests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerExternalTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithCompressionTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithPartitionSubsetTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/TestPartitioner.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/WrongTopicNameTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ZookeeperConfigurationTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaEmbedded.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRule.java diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/AbstractConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/AbstractConfiguration.java new file mode 100644 index 0000000000..b1ed97f897 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/AbstractConfiguration.java @@ -0,0 +1,72 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import java.util.List; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +/** + * Default implementation of a {@link Configuration}, storing the default topic and partitions explicitly. + * + * Implementors must provide a strategy for retrieving the seed brokers. + * + * @author Marius Bogoevici + */ +public abstract class AbstractConfiguration implements InitializingBean, Configuration { + + private List defaultPartitions; + + private String defaultTopic; + + @Override + public void afterPropertiesSet() throws Exception { + Assert.isTrue(CollectionUtils.isEmpty(defaultPartitions) || StringUtils.isEmpty(defaultTopic) + , "A list of default partitions or a default topic may be specified, but not both"); + } + + @Override + public final List getBrokerAddresses() { + return doGetBrokerAddresses(); + } + + protected abstract List doGetBrokerAddresses(); + + @Override + public List getDefaultPartitions() { + return defaultPartitions; + } + + public void setDefaultPartitions(List defaultPartitions) { + this.defaultPartitions = defaultPartitions; + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + public void setDefaultTopic(String defaultTopic) { + this.defaultTopic = defaultTopic; + } + + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddress.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddress.java new file mode 100644 index 0000000000..9bdcc893f7 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddress.java @@ -0,0 +1,102 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import org.springframework.util.StringUtils; + +/** + * Encapsulates the address of a Kafka broker. + * + * @author Marius Bogoevici + */ +public class BrokerAddress { + + public static final int DEFAULT_PORT = 9092; + + private final String host; + + private final int port; + + public BrokerAddress(String host, int port) { + if (StringUtils.isEmpty(host)) { + throw new IllegalArgumentException("Host cannot be empty"); + } + this.host = host; + this.port = port; + } + + public BrokerAddress(String host) { + this(host, DEFAULT_PORT); + } + + public static BrokerAddress fromAddress(String address) { + String[] split = address.split(":"); + if (split.length == 0 || split.length > 2) { + throw new IllegalArgumentException("Expected format [:]"); + } + if (split.length == 2) { + return new BrokerAddress(split[0], Integer.parseInt(split[1])); + } + else { + return new BrokerAddress(split[0]); + } + + } + + public String getHost() { + return host; + } + + public int getPort() { + return port; + } + + @Override + public int hashCode() { + return 31 * host.hashCode() + port; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + BrokerAddress brokerAddress = (BrokerAddress) o; + + if (port != brokerAddress.port) { + return false; + } + if (!host.equals(brokerAddress.host)) { + return false; + } + + return true; + } + + @Override + public String toString() { + return host + ":" + port; + } + +} + + diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddressListConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddressListConfiguration.java new file mode 100644 index 0000000000..e64e007d63 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddressListConfiguration.java @@ -0,0 +1,40 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import java.util.List; + +/** + * Kafka {@link Configuration} where the seed brokers are set up explicitly. + * + * @author Marius Bogoevici + */ +public class BrokerAddressListConfiguration extends AbstractConfiguration { + + private final List brokerAddresses; + + public BrokerAddressListConfiguration(List brokerAddresses) { + this.brokerAddresses = brokerAddresses; + } + + @Override + protected List doGetBrokerAddresses() { + return brokerAddresses; + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Configuration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Configuration.java new file mode 100644 index 0000000000..73c58b7ec0 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Configuration.java @@ -0,0 +1,47 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import java.util.List; + +/** + * Used to configure a {@link DefaultConnectionFactory}. Provides a list of seed brokers. + * + * @author Marius Bogoevici + */ +public interface Configuration { + + /** + * The list of seed broker addresses used by this Configuration. + * @return the broker addresses + */ + List getBrokerAddresses(); + + /** + * A list of default partitions to perform operations on. + * @return the list of partitions + */ + List getDefaultPartitions(); + + /** + * A default topic to perform operations on. + * @return a topic name + */ + String getDefaultTopic(); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Connection.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Connection.java new file mode 100644 index 0000000000..7e69a13c5e --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Connection.java @@ -0,0 +1,91 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import java.util.Map; + +import kafka.api.OffsetRequest; + +/** + * A connection to a Kafka broker. + * + * @author Marius Bogoevici + */ +public interface Connection { + + /** + * Fetch data from a Kafka broker. + * @param fetchRequests a list of fetch operations + * @return message batches, indexed by partition + * @throws ConsumerException the ConsumerException if any underlying error + */ + Result fetch(FetchRequest... fetchRequests) throws ConsumerException; + + /** + * Fetch an actual offset in the partition, immediately before the given reference time, + * or the smallest and largest value, respectively, if the special values -1 + * ({@link OffsetRequest#LatestTime()}) and -2 ({@link OffsetRequest#LatestTime()}) + * are used . To be used to position the initial offset of a read operation. + * @param referenceTime The returned values will be before this time, if they exist. The special + * values -2 ({@link OffsetRequest#LatestTime()}) + * and -1 ({@link OffsetRequest#LatestTime()}) are supported. + * @param partitions the offsets, indexed by {@link Partition} + * @return any errors, an empty {@link Result} in case of success + * @throws ConsumerException the ConsumerException if any underlying error + */ + Result fetchInitialOffset(long referenceTime, Partition... partitions) throws ConsumerException; + + /** + * Fetch offsets from the native Kafka offset management system. + * @param consumerId the id of the consumer + * @param partitions the list of partitions whose offsets are queried for + * @return any errors, an empty {@link Result} in case of success + * @throws ConsumerException the ConsumerException if any underlying error + */ + Result fetchStoredOffsetsForConsumer(String consumerId, Partition... partitions) throws ConsumerException; + + /** + * Update offsets in the native Kafka offset management system. + * @param consumerId the id of the consumer + * @param offsets the offsets, indexed by {@link Partition} + * @return any errors, an empty {@link Result} in case of success + * @throws ConsumerException the ConsumerException if any underlying error + */ + Result commitOffsetsForConsumer(String consumerId, Map offsets) throws ConsumerException; + + /** + * Retrieve the leader broker addresses for all the partitions in the given topics. + * @param topics the topics whose partitions we query for + * @return broker addresses, indexed by {@link Partition} + * @throws ConsumerException the ConsumerException if any underlying error + */ + Result findLeaders(String... topics) throws ConsumerException; + + + /** + * The broker address for this consumer + * @return broker address + */ + BrokerAddress getBrokerAddress(); + + /** + * Closes the connection to the broker. No further operations are permitted. + */ + void close(); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConnectionFactory.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConnectionFactory.java new file mode 100644 index 0000000000..3791ca8061 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConnectionFactory.java @@ -0,0 +1,67 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import java.util.Collection; +import java.util.Map; + +/** + * Creates Kafka connections and retrieves metadata for topics and partitions. + * + * @author Marius Bogoevici + * + */ +public interface ConnectionFactory { + + /** + * Create a connection to a Kafka broker, caching it internally + * @param brokerAddress a broker address + * @return a working connection + */ + Connection connect(BrokerAddress brokerAddress); + + /** + * Retrieve the leaders for a set of partitions. + * @param partitions whose leaders are queried + * @return the broker associated with the provided topic and partition + */ + Map getLeaders(Iterable partitions); + + /** + * Return the leader for a single partition + * @param partition the partition whose leader is queried + * @return the leader's address + */ + BrokerAddress getLeader(Partition partition); + + /** + * Refresh the broker connections and partition leader map. To be called when the topology changes + * are detected (i.e. brokers leave and/or partition leaders change) and that results in fetch errors, + * for instance. + * @param topics the topics for which to refresh the leaders + */ + void refreshLeaders(Collection topics); + + /** + * Retrieves the partitions of a given topic + * @param topic the topic to query for + * @return a list of partitions + */ + Collection getPartitions(String topic); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConsumerException.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConsumerException.java new file mode 100644 index 0000000000..49d9698990 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConsumerException.java @@ -0,0 +1,41 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import kafka.javaapi.consumer.SimpleConsumer; + +/** + * Wraps exceptions thrown by {@link SimpleConsumer} calls. + * + * Because the client is originally written in Scala, checked exceptions are missing from the method signatures + * and cannot be caught directly. + * + * @author Marius Bogoevici + */ +@SuppressWarnings("serial") +public class ConsumerException extends RuntimeException { + + public ConsumerException(Throwable cause) { + super(cause); + } + + public ConsumerException(String message) { + super(message); + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnection.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnection.java new file mode 100644 index 0000000000..4c6b0191aa --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnection.java @@ -0,0 +1,322 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.block.function.Function2; +import com.gs.collections.api.tuple.Pair; +import com.gs.collections.impl.list.mutable.FastList; +import com.gs.collections.impl.tuple.Tuples; +import com.gs.collections.impl.utility.LazyIterate; +import com.gs.collections.impl.utility.MapIterate; +import kafka.api.FetchRequestBuilder; +import kafka.api.PartitionOffsetRequestInfo; +import kafka.cluster.Broker; +import kafka.common.ErrorMapping; +import kafka.common.OffsetMetadataAndError; +import kafka.common.TopicAndPartition; +import kafka.javaapi.FetchResponse; +import kafka.javaapi.OffsetCommitRequest; +import kafka.javaapi.OffsetCommitResponse; +import kafka.javaapi.OffsetFetchRequest; +import kafka.javaapi.OffsetFetchResponse; +import kafka.javaapi.OffsetRequest; +import kafka.javaapi.OffsetResponse; +import kafka.javaapi.PartitionMetadata; +import kafka.javaapi.TopicMetadata; +import kafka.javaapi.TopicMetadataRequest; +import kafka.javaapi.TopicMetadataResponse; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.javaapi.message.ByteBufferMessageSet; +import kafka.message.MessageAndOffset; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.util.Assert; + +/** + * A connection to a Kafka broker. + * + * @author Marius Bogoevici + */ +public class DefaultConnection implements Connection { + + private static Log log = LogFactory.getLog(DefaultConnection.class); + + private final AtomicInteger correlationIdCounter = new AtomicInteger(new Random(new Date().getTime()).nextInt()); + + private final SimpleConsumer simpleConsumer; + + private final BrokerAddress brokerAddress; + + private int minBytes; + + private int maxWait; + + public DefaultConnection(BrokerAddress brokerAddress, String clientId, int bufferSize, int soTimeout, + int minBytes, int maxWait) { + this.brokerAddress = brokerAddress; + this.minBytes = minBytes; + this.maxWait = maxWait; + this.simpleConsumer = + new SimpleConsumer(brokerAddress.getHost(), brokerAddress.getPort(), soTimeout, bufferSize, clientId); + } + + /** + * @see Connection#getBrokerAddress() + */ + @Override + public BrokerAddress getBrokerAddress() { + return brokerAddress; + } + + @Override + public void close() { + this.simpleConsumer.close(); + } + + /** + * Fetche data from Kafka. + * @return a combination of messages and errors, depending on whether the invocation was successful or not + * @throws ConsumerException the ConsumerException if any underlying error + */ + @Override + public Result fetch(FetchRequest... requests) throws ConsumerException { + FetchRequestBuilder fetchRequestBuilder = new FetchRequestBuilder(); + for (FetchRequest request : requests) { + Partition partition = request.getPartition(); + long offset = request.getOffset(); + int maxSize = request.getMaxSizeInBytes(); + fetchRequestBuilder.addFetch(partition.getTopic(), partition.getId(), offset, maxSize); + } + FetchResponse fetchResponse; + try { + fetchResponse = this.simpleConsumer.fetch(fetchRequestBuilder.maxWait(maxWait).minBytes(minBytes).build()); + } + catch (Exception e) { + throw new ConsumerException(e); + } + ResultBuilder resultBuilder = new ResultBuilder(); + for (final FetchRequest request : requests) { + Partition partition = request.getPartition(); + if (log.isDebugEnabled()) { + log.debug("Reading from " + partition + "@" + request.getOffset()); + } + short errorCode = fetchResponse.errorCode(partition.getTopic(), partition.getId()); + if (ErrorMapping.NoError() == errorCode) { + ByteBufferMessageSet messageSet = fetchResponse.messageSet(partition.getTopic(), partition.getId()); + List kafkaMessages = LazyIterate.collect(messageSet, + new ConvertToKafkaMessageFunction(request)).toList(); + long highWatermark = fetchResponse.highWatermark(partition.getTopic(), partition.getId()); + resultBuilder.add(partition).withResult(new KafkaMessageBatch(partition, kafkaMessages, highWatermark)); + } + else { + resultBuilder.add(partition).withError(errorCode); + } + } + return resultBuilder.build(); + } + + @Override + public Result fetchStoredOffsetsForConsumer(String consumerId, Partition... partitions) throws + ConsumerException { + FastList topicsAndPartitions = FastList.newList(Arrays.asList(partitions)) + .collect(new ConvertToTopicAndPartitionFunction()); + OffsetFetchRequest offsetFetchRequest = new OffsetFetchRequest(consumerId, topicsAndPartitions, + kafka.api.OffsetFetchRequest.CurrentVersion(), createCorrelationId(), simpleConsumer.clientId()); + OffsetFetchResponse offsetFetchResponse = null; + try { + offsetFetchResponse = simpleConsumer.fetchOffsets(offsetFetchRequest); + } + catch (Exception e) { + throw new ConsumerException(e); + } + ResultBuilder resultBuilder = new ResultBuilder(); + for (Partition partition : partitions) { + OffsetMetadataAndError offsetMetadataAndError + = offsetFetchResponse.offsets().get(new TopicAndPartition(partition.getTopic(), partition.getId())); + short errorCode = offsetMetadataAndError.error(); + if (ErrorMapping.NoError() == errorCode) { + resultBuilder.add(partition).withResult(offsetMetadataAndError.offset()); + } + else { + resultBuilder.add(partition).withError(errorCode); + } + } + return resultBuilder.build(); + } + + /** + * @see Connection#fetchInitialOffset(long, Partition...) + */ + @Override + public Result fetchInitialOffset(long referenceTime, Partition... partitions) throws + ConsumerException { + Assert.isTrue(partitions.length > 0, "Must provide at least one partition"); + Map infoMap = new HashMap(); + for (Partition partition : partitions) { + infoMap.put(new TopicAndPartition(partition.getTopic(), partition.getId()), + new PartitionOffsetRequestInfo(referenceTime, 1)); + } + OffsetRequest offsetRequest = + new OffsetRequest(infoMap, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId()); + OffsetResponse offsetResponse = null; + try { + offsetResponse = simpleConsumer.getOffsetsBefore(offsetRequest); + } + catch (Exception e) { + throw new ConsumerException(e); + } + ResultBuilder resultBuilder = new ResultBuilder(); + for (Partition partition : partitions) { + short errorCode = offsetResponse.errorCode(partition.getTopic(), partition.getId()); + if (ErrorMapping.NoError() == errorCode) { + long[] offsets = offsetResponse.offsets(partition.getTopic(), partition.getId()); + if (offsets.length == 0) { + // normally, we shouldn't get here - either an offset has been returned, + // or an error. However, in case something went wrong, the check protects against an + // ArrayIndexOutOfBoundsException + throw new ConsumerException("Inconsistent response: no error has been returned, " + + "but no offsets either"); + } + resultBuilder.add(partition).withResult(offsets[0]); + } + else { + resultBuilder.add(partition).withError(errorCode); + } + } + return resultBuilder.build(); + } + + @Override + public Result commitOffsetsForConsumer(String consumerId, Map offsets) + throws ConsumerException { + Map requestInfo = + MapIterate.collect(offsets, new CreateRequestInfoMapEntryFunction()); + OffsetCommitResponse offsetCommitResponse = null; + try { + offsetCommitResponse = simpleConsumer.commitOffsets( + new OffsetCommitRequest(consumerId, requestInfo, kafka.api.OffsetCommitRequest.CurrentVersion(), + createCorrelationId(), simpleConsumer.clientId())); + } + catch (Exception e) { + throw new ConsumerException(e); + } + ResultBuilder resultBuilder = new ResultBuilder(); + for (TopicAndPartition topicAndPartition : requestInfo.keySet()) { + if (offsetCommitResponse.errors().containsKey(topicAndPartition)) { + Partition partition = new Partition(topicAndPartition.topic(), topicAndPartition.partition()); + resultBuilder.add(partition).withError((Short) offsetCommitResponse.errors().get(topicAndPartition)); + } + } + return resultBuilder.build(); + } + + /** + * @see Connection#findLeaders(String...) + */ + @Override + public Result findLeaders(String... topics) throws ConsumerException { + TopicMetadataRequest topicMetadataRequest = + new TopicMetadataRequest(Arrays.asList(topics), createCorrelationId()); + TopicMetadataResponse topicMetadataResponse = null; + try { + topicMetadataResponse = simpleConsumer.send(topicMetadataRequest); + } + catch (Exception e) { + throw new ConsumerException(e); + } + ResultBuilder resultBuilder = new ResultBuilder(); + for (TopicMetadata topicMetadata : topicMetadataResponse.topicsMetadata()) { + if (topicMetadata.errorCode() != ErrorMapping.NoError()) { + resultBuilder.add(new Partition(topicMetadata.topic(), -1)).withError(topicMetadata.errorCode()); + } + else { + for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) { + Partition partition = new Partition(topicMetadata.topic(), partitionMetadata.partitionId()); + if (ErrorMapping.NoError() == partitionMetadata.errorCode()) { + Broker leader = partitionMetadata.leader(); + BrokerAddress result = new BrokerAddress(leader.host(), leader.port()); + resultBuilder.add(partition).withResult(result); + } + else { + resultBuilder.add(partition).withError(partitionMetadata.errorCode()); + } + } + + } + } + return resultBuilder.build(); + } + + + + /** + * Creat a pseudo-unique correlation id for requests and responses + * @return correlation id + */ + private Integer createCorrelationId() { + return correlationIdCounter.incrementAndGet(); + } + + @SuppressWarnings("serial") + private static class ConvertToKafkaMessageFunction implements Function { + private final FetchRequest request; + + public ConvertToKafkaMessageFunction(FetchRequest request) { + this.request = request; + } + + @Override + public KafkaMessage valueOf(MessageAndOffset messageAndOffset) { + return new KafkaMessage(messageAndOffset.message(), + new KafkaMessageMetadata(request.getPartition(), messageAndOffset.offset(), + messageAndOffset.nextOffset())); + } + } + + @SuppressWarnings("serial") + private static class ConvertToTopicAndPartitionFunction implements Function { + + @Override + public TopicAndPartition valueOf(Partition partition) { + return new TopicAndPartition(partition.getTopic(), partition.getId()); + } + + } + + @SuppressWarnings("serial") + private static class CreateRequestInfoMapEntryFunction + implements Function2> { + + @Override + public Pair value(Partition partition, Long offset) { + return Tuples.pair(new TopicAndPartition(partition.getTopic(), partition.getId()), + new OffsetMetadataAndError(offset, OffsetMetadataAndError.NoMetadata(), ErrorMapping.NoError())); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnectionFactory.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnectionFactory.java new file mode 100644 index 0000000000..922e04519f --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnectionFactory.java @@ -0,0 +1,260 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.impl.block.factory.Functions; +import com.gs.collections.impl.list.mutable.FastList; +import com.gs.collections.impl.map.mutable.UnifiedMap; +import com.gs.collections.impl.utility.ListIterate; +import kafka.client.ClientUtils$; +import kafka.common.KafkaException; +import kafka.javaapi.PartitionMetadata; +import kafka.javaapi.TopicMetadata; +import kafka.javaapi.TopicMetadataResponse; +import scala.collection.JavaConversions; + +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.context.SmartLifecycle; +import org.springframework.util.Assert; + +/** + * Default implementation of {@link ConnectionFactory} + * + * @author Marius Bogoevici + */ +public class DefaultConnectionFactory implements InitializingBean, ConnectionFactory, DisposableBean { + + private final GetBrokersByPartitionFunction getBrokersByPartitionFunction = new GetBrokersByPartitionFunction(); + + private final ConnectionInstantiationFunction connectionInstantiationFunction = new ConnectionInstantiationFunction(); + + private final Configuration configuration; + + private final AtomicReference partitionBrokerMapReference = new AtomicReference(); + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + + private String clientId = KafkaConsumerDefaults.GROUP_ID; + + private int minBytes = KafkaConsumerDefaults.MIN_FETCH_BYTES; + + private int maxWait = KafkaConsumerDefaults.MAX_WAIT_TIME_IN_MS; + + private int bufferSize = KafkaConsumerDefaults.SOCKET_BUFFER_SIZE_INT; + + private int socketTimeout = KafkaConsumerDefaults.SOCKET_TIMEOUT_INT; + + private int fetchMetadataTimeout = KafkaConsumerDefaults.FETCH_METADATA_TIMEOUT; + + private final UnifiedMap kafkaBrokersCache = UnifiedMap.newMap(); + + public DefaultConnectionFactory(Configuration configuration) { + this.configuration = configuration; + } + + public Configuration getConfiguration() { + return configuration; + } + + @Override + public void afterPropertiesSet() throws Exception { + Assert.notNull(this.configuration, "Kafka configuration cannot be empty"); + this.refreshLeaders(configuration.getDefaultTopic() == null ? Collections.emptyList() : + Collections.singletonList(configuration.getDefaultTopic())); + } + + @Override + public void destroy() throws Exception { + for (Connection connection : kafkaBrokersCache) { + connection.close(); + } + } + + /** + * The minimum amount of data that a server fetch operation will wait for before returning, + * unless {@code maxWaitTimeInMs} has elapsed. + * In conjunction with {@link DefaultConnectionFactory#setMaxWait(int)}, controls latency + * and throughput. + * Smaller values increase responsiveness, but may increase the number of poll operations, + * potentially reducing throughput and increasing CPU consumption. + * @param minBytes the amount of data to fetch + */ + public void setMinBytes(int minBytes) { + this.minBytes = minBytes; + } + + /** + * The maximum amount of time that a server fetch operation will wait before returning + * (unless {@code minFetchSizeInBytes}) are available. + * In conjunction with {@link DefaultConnectionFactory#setMinBytes(int)}, + * controls latency and throughput. + * Smaller intervals increase responsiveness, but may increase + * the number of poll operations, potentially increasing CPU + * consumption and reducing throughput. + * @param maxWait timeout to wait + */ + public void setMaxWait(int maxWait) { + this.maxWait = maxWait; + } + + public String getClientId() { + return clientId; + } + + /** + * A client name to be used throughout this connection. + * @param clientId the client name + */ + public void setClientId(String clientId) { + this.clientId = clientId; + } + + /** + * The buffer size for this client + * @param bufferSize the buffer size + */ + public void setBufferSize(int bufferSize) { + this.bufferSize = bufferSize; + } + + /** + * The socket timeout for this client + * @param socketTimeout the socket timeout + */ + public void setSocketTimeout(int socketTimeout) { + this.socketTimeout = socketTimeout; + } + + /** + * The timeout on fetching metadata (e.g. partition leaders) + * @param fetchMetadataTimeout timeout + */ + public void setFetchMetadataTimeout(int fetchMetadataTimeout) { + this.fetchMetadataTimeout = fetchMetadataTimeout; + } + + /** + * @see ConnectionFactory#getLeaders(Iterable) + */ + @Override + public Map getLeaders(Iterable partitions) { + return FastList.newList(partitions).toMap(Functions.getPassThru(), getBrokersByPartitionFunction); + } + + /** + * @see ConnectionFactory#getLeader(Partition) + */ + @Override + public BrokerAddress getLeader(Partition partition) { + try { + lock.readLock().lock(); + return this.getLeaders(Collections.singleton(partition)).get(partition); + } + finally { + lock.readLock().unlock(); + } + } + + /** + * @see ConnectionFactory#connect(BrokerAddress) + */ + @Override + public Connection connect(BrokerAddress brokerAddress) { + return kafkaBrokersCache.getIfAbsentPutWithKey(brokerAddress, connectionInstantiationFunction); + } + + /** + * @see ConnectionFactory#refreshLeaders(Collection) + */ + @Override + public void refreshLeaders(Collection topics) { + try { + lock.writeLock().lock(); + for (Connection connection : kafkaBrokersCache) { + connection.close(); + } + String brokerAddressesAsString = + ListIterate.collect(configuration.getBrokerAddresses(), Functions.getToString()) + .makeString(","); + TopicMetadataResponse topicMetadataResponse = + new TopicMetadataResponse( + ClientUtils$.MODULE$.fetchTopicMetadata( + JavaConversions.asScalaSet(new HashSet(topics)), + ClientUtils$.MODULE$.parseBrokerList(brokerAddressesAsString), + getClientId(), fetchMetadataTimeout, 0)); + Map kafkaBrokerAddressMap = new HashMap(); + for (TopicMetadata topicMetadata : topicMetadataResponse.topicsMetadata()) { + for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) { + kafkaBrokerAddressMap.put(new Partition(topicMetadata.topic(), partitionMetadata.partitionId()), + new BrokerAddress(partitionMetadata.leader().host(), partitionMetadata.leader().port())); + } + } + this.partitionBrokerMapReference.set(new PartitionBrokerMap(UnifiedMap.newMap(kafkaBrokerAddressMap))); + } + finally { + lock.writeLock().unlock(); + } + } + + /** + * @see ConnectionFactory#getPartitions(String) + */ + @Override + public Collection getPartitions(String topic) { + if (!getPartitionBrokerMap().getPartitionsByTopic().containsKey(topic)) { + throw new TopicNotFoundException(topic); + } + return getPartitionBrokerMap().getPartitionsByTopic().get(topic).toList(); + } + + private PartitionBrokerMap getPartitionBrokerMap() { + return partitionBrokerMapReference.get(); + } + + @SuppressWarnings("serial") + private class ConnectionInstantiationFunction implements Function { + + @Override + public Connection valueOf(BrokerAddress brokerAddress) { + return new DefaultConnection(brokerAddress, clientId, bufferSize, socketTimeout, minBytes, maxWait); + } + + } + + @SuppressWarnings("serial") + private class GetBrokersByPartitionFunction implements Function { + + @Override + public BrokerAddress valueOf(Partition partition) { + return partitionBrokerMapReference.get().getBrokersByPartition().get(partition); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/FetchRequest.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/FetchRequest.java new file mode 100644 index 0000000000..a3b523f9e6 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/FetchRequest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +/** + * Encapsulates a request for fetching messages from the server. + * + * @author Marius Bogoevici + */ +public class FetchRequest { + + private Partition partition; + + private long offset; + + private int maxSizeInBytes; + + public FetchRequest(Partition Partition, long offset, int maxSizeInBytes) { + this.partition = Partition; + this.offset = offset; + this.maxSizeInBytes = maxSizeInBytes; + } + + public Partition getPartition() { + return partition; + } + + public void setPartition(Partition partition) { + this.partition = partition; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } + + public int getMaxSizeInBytes() { + return maxSizeInBytes; + } + + public void setMaxSizeInBytes(int maxSizeInBytes) { + this.maxSizeInBytes = maxSizeInBytes; + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java index e258f90cd2..2b332b7ca4 100644 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2013 the original author or authors. + * Copyright 2002-2015 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,29 +13,56 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.springframework.integration.kafka.core; +import kafka.api.OffsetRequest; + /** * Kafka adapter specific message headers. * * @author Soby Chacko + * @author Marius Bogoevici * @since 0.5 */ -public final class KafkaConsumerDefaults { +public abstract class KafkaConsumerDefaults { + //High level consumer public static final String GROUP_ID = "groupid"; - public static final String SOCKET_TIMEOUT = "30000"; - public static final String SOCKET_BUFFER_SIZE = "64*1024"; - public static final String FETCH_SIZE = "300 * 1024"; + + public static final int SOCKET_TIMEOUT_INT = 30000; + + public static final String SOCKET_TIMEOUT = Integer.toString(SOCKET_TIMEOUT_INT); + + public static final int SOCKET_BUFFER_SIZE_INT = 64*1024; + + public static final String SOCKET_BUFFER_SIZE = Integer.toString(SOCKET_BUFFER_SIZE_INT); + + public static final int FETCH_SIZE_INT = 300 * 1024; + + public static final String FETCH_SIZE = Integer.toString(FETCH_SIZE_INT); + public static final String BACKOFF_INCREMENT = "1000"; + public static final String QUEUED_CHUNKS_MAX = "100"; + public static final String AUTO_COMMIT_ENABLE = "true"; + public static final String AUTO_COMMIT_INTERVAL = "10000"; + public static final String AUTO_OFFSET_RESET = "smallest"; + //Overriding the default value of -1, which will make the consumer to wait indefinitely public static final String CONSUMER_TIMEOUT = "5000"; + public static final String REBALANCE_RETRIES_MAX = "4"; - private KafkaConsumerDefaults() { - } + public static final int MIN_FETCH_BYTES = 1; + + public static final int MAX_WAIT_TIME_IN_MS = 100; + + public static final long DEFAULT_OFFSET_RESET = OffsetRequest.EarliestTime(); + + public static final int FETCH_METADATA_TIMEOUT = 10000; + } diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessage.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessage.java new file mode 100644 index 0000000000..cbf8d19851 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessage.java @@ -0,0 +1,53 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import kafka.message.Message; + +import org.springframework.util.ObjectUtils; + +/** + * Wrapper around a {@link Message kafka message} and its {@link KafkaMessageMetadata metadata}. + * + * @author Marius Bogoevici + */ +public class KafkaMessage { + + private final Message message; + + private final KafkaMessageMetadata metadata; + + public KafkaMessage(Message message, KafkaMessageMetadata metadata) { + this.message = message; + this.metadata = metadata; + } + + public Message getMessage() { + return message; + } + + public KafkaMessageMetadata getMetadata() { + return metadata; + } + + @Override + public String toString() { + return "KafkaMessage [" + ObjectUtils.nullSafeToString(message) + ", " + + ObjectUtils.nullSafeToString(metadata.toString()) + "]"; + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageBatch.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageBatch.java new file mode 100644 index 0000000000..8e718cf0b0 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageBatch.java @@ -0,0 +1,64 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import java.util.List; + +/** + * A group of {@link KafkaMessage}s retrieved by a fetch operation + * + * @author Marius Bogoevici + */ +public class KafkaMessageBatch { + + private Partition partition; + + private List messages; + + private long highWatermark; + + public KafkaMessageBatch(Partition partition, List messages, long highWatermark) { + this.partition = partition; + this.messages = messages; + this.highWatermark = highWatermark; + } + + public Partition getPartition() { + return partition; + } + + public void setPartition(Partition partition) { + this.partition = partition; + } + + public List getMessages() { + return messages; + } + + public void setMessages(List messages) { + this.messages = messages; + } + + public long getHighWatermark() { + return highWatermark; + } + + public void setHighWatermark(long highWatermark) { + this.highWatermark = highWatermark; + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageMetadata.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageMetadata.java new file mode 100644 index 0000000000..aac33c6050 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageMetadata.java @@ -0,0 +1,65 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import kafka.message.Message; + +import org.springframework.util.Assert; + +/** + * Metadata for a Kafka {@link Message}. + * + * @author Marius Bogoevici + */ +public class KafkaMessageMetadata { + + private final long offset; + + private final long nextOffset; + + private final Partition partition; + + public KafkaMessageMetadata(Partition partition, long offset, long nextOffset) { + Assert.notNull(partition); + Assert.isTrue(offset >= 0); + Assert.isTrue(nextOffset >= 0); + this.offset = offset; + this.nextOffset = nextOffset; + this.partition = partition; + } + + public Partition getPartition() { + return partition; + } + + public long getOffset() { + return offset; + } + + public long getNextOffset() { + return nextOffset; + } + + @Override + public String toString() { + return "KafkaMessageMetadata [" + + "offset=" + offset + + ", nextOffset=" + nextOffset + + ", " + partition.toString(); + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaOperations.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaOperations.java new file mode 100644 index 0000000000..4d6a198c35 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaOperations.java @@ -0,0 +1,31 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +/** + * @author Marius Bogoevici + */ +public interface KafkaOperations { + + /** + * Receive data from the server. + * @param messageFetchRequests the set of FetchRequest + * @return the Result with KafkaMessageBatch + */ + Result receive(Iterable messageFetchRequests); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaTemplate.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaTemplate.java new file mode 100644 index 0000000000..659abb1a87 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaTemplate.java @@ -0,0 +1,65 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.list.MutableList; +import com.gs.collections.impl.list.mutable.FastList; + + +/** + * A template for executing high-level operations on a set of Kafka brokers. + * + * @author Marius Bogoevici + */ +public class KafkaTemplate implements KafkaOperations { + + private final ConnectionFactory connectionFactory; + + private FetchRequestToLeaderBrokerAddress fetchRequestToLeaderBrokerAddress = new FetchRequestToLeaderBrokerAddress(); + + public KafkaTemplate(ConnectionFactory connectionFactory) { + this.connectionFactory = connectionFactory; + } + + public ConnectionFactory getConnectionFactory() { + return connectionFactory; + } + + @Override + public Result receive(Iterable messageFetchRequests) { + FastList requestList = FastList.newList(messageFetchRequests); + MutableList distinctBrokerAddresses = + requestList.collect(fetchRequestToLeaderBrokerAddress).distinct(); + if (distinctBrokerAddresses.size() != 1) { + throw new IllegalArgumentException("All messages must be fetched from the same broker"); + } + return connectionFactory.connect(distinctBrokerAddresses.getFirst()) + .fetch(requestList.toTypedArray(FetchRequest.class)); + } + + @SuppressWarnings("serial") + private class FetchRequestToLeaderBrokerAddress implements Function { + + @Override + public BrokerAddress valueOf(FetchRequest fetchRequest) { + return connectionFactory.getLeader(fetchRequest.getPartition()); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Partition.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Partition.java new file mode 100644 index 0000000000..db19f4b3c7 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Partition.java @@ -0,0 +1,81 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +/** + * A reference to a Kafka partition, with both topic and partition id + * + * @author Marius Bogoevici + */ +public class Partition { + + private String topic; + + private int id; + + public Partition(String topic, int id) { + this.topic = topic; + this.id = id; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + @Override + public int hashCode() { + int result = topic.hashCode(); + result = 31 * result + id; + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Partition partition = (Partition) o; + if (id != partition.id) { + return false; + } + if (!topic.equals(partition.topic)) { + return false; + } + return true; + } + + @Override + public String toString() { + return "Partition[" + "topic='" + topic + '\'' + ", id=" + id + ']'; + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/PartitionBrokerMap.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/PartitionBrokerMap.java new file mode 100644 index 0000000000..6a7c0276a3 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/PartitionBrokerMap.java @@ -0,0 +1,66 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.map.ImmutableMap; +import com.gs.collections.api.multimap.Multimap; +import com.gs.collections.impl.map.mutable.UnifiedMap; + +/** + * Immutable store for the partition/broker mapping + * + * @author Marius Bogoevici + */ +class PartitionBrokerMap { + + private static final GetTopicFunction getTopicFunction = new GetTopicFunction(); + + private final Multimap partitionsByBroker; + + private final ImmutableMap brokersByPartition; + + private final Multimap partitionsByTopic; + + public PartitionBrokerMap(UnifiedMap brokersByPartition) { + this.brokersByPartition = brokersByPartition.toImmutable(); + this.partitionsByTopic = this.brokersByPartition.keysView().groupBy(getTopicFunction); + this.partitionsByBroker = brokersByPartition.flip().toImmutable(); + } + + public Multimap getPartitionsByBroker() { + return partitionsByBroker; + } + + public ImmutableMap getBrokersByPartition() { + return brokersByPartition; + } + + public Multimap getPartitionsByTopic() { + return partitionsByTopic; + } + + @SuppressWarnings("serial") + private static class GetTopicFunction implements Function { + + @Override + public String valueOf(Partition partition) { + return partition.getTopic(); + } + + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Result.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Result.java new file mode 100644 index 0000000000..a2544a95f0 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Result.java @@ -0,0 +1,64 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import java.util.Collections; +import java.util.Map; + +/** + * The result of a Kafka SimpleConsumer operation + * + * @author Marius Bogoevici + */ +public class Result { + + private final Map results; + + private final Map errors; + + Result(Map results, Map errors) { + this.results = Collections.unmodifiableMap(results); + this.errors = Collections.unmodifiableMap(errors); + } + + public Map getResults() { + return results; + } + + public T getResult(Partition partition) throws IllegalArgumentException { + if (this.results.containsKey(partition)) { + return this.results.get(partition); + } + else { + throw new IllegalArgumentException(" No result received for " + partition.toString()); + } + } + + public Map getErrors() { + return errors; + } + + public short getError(Partition partition) throws IllegalArgumentException { + if (this.getErrors().containsKey(partition)) { + return this.getErrors().get(partition); + } + else { + throw new IllegalArgumentException("No error received for " + partition.toString()); + } + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ResultBuilder.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ResultBuilder.java new file mode 100644 index 0000000000..fd2220d3ad --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ResultBuilder.java @@ -0,0 +1,74 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import java.util.HashMap; +import java.util.Map; + +/** + * Utility class for building a {@link Result} + * + * @author Marius Bogoevici + */ +class ResultBuilder { + + private Map result; + + private Map errors; + + public ResultBuilder() { + this.result = new HashMap(); + this.errors = new HashMap(); + } + + public KafkaPartitionResultHolder add(Partition Partition) { + return new KafkaPartitionResultHolder(Partition); + } + + public Result build() { + return new Result(result, errors); + } + + class KafkaPartitionResultHolder { + + private Partition partition; + + public KafkaPartitionResultHolder(Partition Partition) { + this.partition = Partition; + } + + public ResultBuilder withResult(T result) { + if (ResultBuilder.this.errors.containsKey(partition)) { + throw new IllegalArgumentException("A KafkaResult cannot contain both an error " + + "and a result for the same topic and partition"); + } + ResultBuilder.this.result.put(partition, result); + return ResultBuilder.this; + } + + public ResultBuilder withError(short error) { + if (ResultBuilder.this.result.containsKey(partition)) { + throw new IllegalArgumentException("A FetchResult cannot contain both an error " + + "and a MessageSet for the same topic and partition"); + } + ResultBuilder.this.errors.put(partition, error); + return ResultBuilder.this; + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/TopicNotFoundException.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/TopicNotFoundException.java new file mode 100644 index 0000000000..2d99c0c2cf --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/TopicNotFoundException.java @@ -0,0 +1,29 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +/** + * @author Marius Bogoevici + */ +@SuppressWarnings("serial") +public class TopicNotFoundException extends ConsumerException { + + public TopicNotFoundException(String topicName) { + super(String.format("No topic named '%s' found", topicName)); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConfiguration.java new file mode 100644 index 0000000000..5acca80c33 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConfiguration.java @@ -0,0 +1,99 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import java.util.List; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.impl.list.mutable.FastList; +import kafka.cluster.Broker; +import kafka.utils.ZKStringSerializer$; +import kafka.utils.ZkUtils$; +import org.I0Itec.zkclient.ZkClient; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import scala.collection.JavaConversions; +import scala.collection.Seq; + +import org.springframework.beans.factory.BeanInitializationException; +import org.springframework.integration.kafka.support.ZookeeperConnect; + +/** + * Kafka {@link Configuration} that uses a ZooKeeper connection for retrieving the list of seed brokers. + * + * @author Marius Bogoevici + */ +public class ZookeeperConfiguration extends AbstractConfiguration { + + private final static Log log = LogFactory.getLog(ZookeeperConfiguration.class); + + public static final BrokerToBrokerAddressFunction brokerToBrokerAddressFunction = new BrokerToBrokerAddressFunction(); + + private String zookeeperServers; + + private int sessionTimeout; + + private int connectionTimeout; + + public ZookeeperConfiguration(ZookeeperConnect zookeeperConnect) { + this.zookeeperServers = zookeeperConnect.getZkConnect(); + try { + this.sessionTimeout = Integer.parseInt(zookeeperConnect.getZkSessionTimeout()); + } + catch (NumberFormatException e) { + throw new BeanInitializationException("Cannot parse session timeout:", e); + } + try { + this.connectionTimeout = Integer.parseInt(zookeeperConnect.getZkConnectionTimeout()); + } + catch (NumberFormatException e) { + throw new BeanInitializationException("Cannot parse connection timeout:", e); + } + } + + @Override + protected List doGetBrokerAddresses() { + ZkClient zkClient = null; + try { + zkClient = new ZkClient(zookeeperServers, sessionTimeout, connectionTimeout, ZKStringSerializer$.MODULE$); + Seq allBrokersInCluster = ZkUtils$.MODULE$.getAllBrokersInCluster(zkClient); + FastList brokers = FastList.newList(JavaConversions.asJavaCollection(allBrokersInCluster)); + return brokers.collect(brokerToBrokerAddressFunction); + } + finally { + if (zkClient != null) { + try { + zkClient.close(); + } + catch (Exception e) { + log.error("Cannot close Zookeeper client: ", e); + } + } + } + } + + @SuppressWarnings("serial") + private static class BrokerToBrokerAddressFunction implements Function { + + @Override + public BrokerAddress valueOf(Broker broker) { + return new BrokerAddress(broker.host(), broker.port()); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaMessageDrivenChannelAdapter.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaMessageDrivenChannelAdapter.java new file mode 100644 index 0000000000..6115ae25d0 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaMessageDrivenChannelAdapter.java @@ -0,0 +1,112 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.inbound; + +import kafka.serializer.Decoder; +import kafka.serializer.DefaultDecoder; + +import org.springframework.integration.context.OrderlyShutdownCapable; +import org.springframework.integration.endpoint.MessageProducerSupport; +import org.springframework.integration.kafka.core.KafkaMessageMetadata; +import org.springframework.integration.kafka.listener.AbstractDecodingMessageListener; +import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer; +import org.springframework.integration.kafka.support.KafkaHeaders; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.util.Assert; + +/** + * @author Marius Bogoevici + */ +public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport implements OrderlyShutdownCapable { + + private KafkaMessageListenerContainer messageListenerContainer; + + private Decoder keyDecoder = new DefaultDecoder(null); + + private Decoder payloadDecoder = new DefaultDecoder(null); + + public KafkaMessageDrivenChannelAdapter(KafkaMessageListenerContainer messageListenerContainer) { + Assert.notNull(messageListenerContainer); + Assert.isNull(messageListenerContainer.getMessageListener()); + this.messageListenerContainer = messageListenerContainer; + this.messageListenerContainer.setAutoStartup(false); + } + + public void setKeyDecoder(Decoder keyDecoder) { + this.keyDecoder = keyDecoder; + } + + public void setPayloadDecoder(Decoder payloadDecoder) { + this.payloadDecoder = payloadDecoder; + } + + @Override + protected void onInit() { + this.messageListenerContainer.setMessageListener(new ChannelForwardingMessageListener()); + super.onInit(); + } + + @Override + protected void doStart() { + this.messageListenerContainer.start(); + } + + @Override + protected void doStop() { + this.messageListenerContainer.stop(); + } + + @Override + public String getComponentType() { + return "kafka:message-driven-channel-adapter"; + } + + @Override + public int beforeShutdown() { + this.messageListenerContainer.stop(); + return getPhase(); + } + + @Override + public int afterShutdown() { + return getPhase(); + } + + @SuppressWarnings("rawtypes") + private class ChannelForwardingMessageListener extends AbstractDecodingMessageListener { + + @SuppressWarnings("unchecked") + public ChannelForwardingMessageListener() { + super(keyDecoder, payloadDecoder); + } + + @Override + public void doOnMessage(Object key, Object payload, KafkaMessageMetadata metadata) { + Message message = getMessageBuilderFactory() + .withPayload(payload) + .setHeader(KafkaHeaders.MESSAGE_KEY, key) + .setHeader(KafkaHeaders.TOPIC, metadata.getPartition().getTopic()) + .setHeader(KafkaHeaders.PARTITION_ID, metadata.getPartition().getId()) + .setHeader(KafkaHeaders.OFFSET, metadata.getOffset()) + .build(); + KafkaMessageDrivenChannelAdapter.this.sendMessage(message); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingMessageListener.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingMessageListener.java new file mode 100644 index 0000000000..9db10afa38 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingMessageListener.java @@ -0,0 +1,62 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.listener; + + +import kafka.serializer.Decoder; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.KafkaMessageMetadata; +import org.springframework.integration.kafka.util.MessageUtils; +import org.springframework.util.Assert; + +/** + * Base {@link MessageListener} implementation that decodes the key and the payload using the supplied + * {@link Decoder}s. + * + * Users of this class must extend it and implement {@code doOnMessage} and must supply {@link Decoder} + * implementations for both the key and the payload. + * + * @author Marius Bogoevici + */ +public abstract class AbstractDecodingMessageListener implements MessageListener { + + private Decoder keyDecoder; + + private Decoder

payloadDecoder; + + public AbstractDecodingMessageListener(Decoder keyDecoder, Decoder

payloadDecoder) { + this.keyDecoder = keyDecoder; + this.payloadDecoder = payloadDecoder; + } + + @Override + public final void onMessage(KafkaMessage message) { + this.doOnMessage(MessageUtils.decodeKey(message, keyDecoder), + MessageUtils.decodePayload(message, payloadDecoder), message.getMetadata()); + } + + /** + * Process the decoded message + * @param key the message key + * @param payload the message body + * @param metadata the KafkaMessageMetadata + */ + public abstract void doOnMessage(K key, P payload, KafkaMessageMetadata metadata); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ConcurrentMessageListenerDispatcher.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ConcurrentMessageListenerDispatcher.java new file mode 100644 index 0000000000..62bbffe4a1 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ConcurrentMessageListenerDispatcher.java @@ -0,0 +1,157 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.listener; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +import com.gs.collections.api.block.procedure.Procedure; +import com.gs.collections.api.block.procedure.Procedure2; +import com.gs.collections.api.map.MutableMap; +import com.gs.collections.impl.factory.Maps; + +import org.springframework.context.Lifecycle; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.scheduling.concurrent.CustomizableThreadFactory; +import org.springframework.util.Assert; + +/** + * Dispatches {@link KafkaMessage}s to a {@link MessageListener}. Messages may be + * processed concurrently, according to the {@code concurrency} settings, but messages + * from the same partition are being processed in their original order. + * + * @author Marius Bogoevici + */ +class ConcurrentMessageListenerDispatcher implements Lifecycle { + + public static final CustomizableThreadFactory THREAD_FACTORY = new CustomizableThreadFactory("dispatcher-"); + + private static final StartDelegateProcedure startDelegateProcedure = new StartDelegateProcedure(); + + private static final StopDelegateProcedure stopDelegateProcedure = new StopDelegateProcedure(); + + private final Object lifecycleMonitor = new Object(); + + private final Collection partitions; + + private final int consumers; + + private volatile boolean running; + + private final MessageListener delegateListener; + + private final ErrorHandler errorHandler; + + private final OffsetManager offsetManager; + + private MutableMap delegates; + + private final int queueSize; + + private Executor taskExecutor; + + public ConcurrentMessageListenerDispatcher(MessageListener delegateListener, ErrorHandler errorHandler, + Collection partitions, OffsetManager offsetManager, int consumers, int queueSize) { + Assert.notEmpty(partitions, "A set of partitions must be provided"); + Assert.isTrue(consumers <= partitions.size(), + "The number of consumers must be smaller or equal to the number of partitions"); + Assert.notNull(delegateListener, "A delegate must be provided"); + this.delegateListener = delegateListener; + this.errorHandler = errorHandler; + this.partitions = partitions; + this.offsetManager = offsetManager; + this.consumers = consumers; + this.queueSize = queueSize; + } + + @Override + public void start() { + synchronized (lifecycleMonitor) { + if (!isRunning()) { + initializeAndStartDispatching(); + this.running = true; + } + } + } + + @Override + public void stop() { + synchronized (lifecycleMonitor) { + if (isRunning()) { + this.running = false; + delegates.flip().keyBag().toSet().forEach(stopDelegateProcedure); + } + } + } + + @Override + public boolean isRunning() { + return running; + } + + public void dispatch(KafkaMessage message) { + delegates.get(message.getMetadata().getPartition()).enqueue(message); + } + + private void initializeAndStartDispatching() { + // allocate delegate instances index them + List delegateList = new ArrayList(consumers); + for (int i = 0; i < consumers; i++) { + QueueingMessageListenerInvoker blockingQueueMessageListenerInvoker = + new QueueingMessageListenerInvoker(queueSize, offsetManager, delegateListener, errorHandler); + delegateList.add(blockingQueueMessageListenerInvoker); + } + // evenly distribute partitions across delegates + delegates = Maps.mutable.of(); + int i = 0; + for (Partition partition : partitions) { + delegates.put(partition, delegateList.get((i++) % consumers)); + } + // initialize task executor + if (this.taskExecutor == null) { + this.taskExecutor = Executors.newFixedThreadPool(consumers, THREAD_FACTORY); + } + // start dispatchers + delegates.flip().keyBag().toSet().forEachWith(startDelegateProcedure, taskExecutor); + } + + @SuppressWarnings("serial") + private static class StopDelegateProcedure implements Procedure { + + @Override + public void value(QueueingMessageListenerInvoker delegate) { + delegate.stop(); + } + + } + + @SuppressWarnings("serial") + private static class StartDelegateProcedure implements Procedure2 { + + @Override + public void value(QueueingMessageListenerInvoker delegate, Executor executor) { + delegate.start(); + executor.execute(delegate); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ErrorHandler.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ErrorHandler.java new file mode 100644 index 0000000000..259389375b --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ErrorHandler.java @@ -0,0 +1,31 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.springframework.integration.kafka.core.KafkaMessage; + +/** + * Handles errors thrown during the execution of a {@link MessageListener} + * + * @author Marius Bogoevici + */ +public interface ErrorHandler { + + void handle(Exception thrownException, KafkaMessage message); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaMessageListenerContainer.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaMessageListenerContainer.java new file mode 100644 index 0000000000..9337f8d73d --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaMessageListenerContainer.java @@ -0,0 +1,578 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.listener; + +import static com.gs.collections.impl.utility.ArrayIterate.flatCollect; +import static com.gs.collections.impl.utility.Iterate.partition; +import static com.gs.collections.impl.utility.MapIterate.forEachKeyValue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +import com.gs.collections.api.RichIterable; +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.block.predicate.Predicate; +import com.gs.collections.api.block.procedure.Procedure; +import com.gs.collections.api.block.procedure.Procedure2; +import com.gs.collections.api.collection.MutableCollection; +import com.gs.collections.api.list.ImmutableList; +import com.gs.collections.api.list.MutableList; +import com.gs.collections.api.multimap.MutableMultimap; +import com.gs.collections.api.partition.PartitionIterable; +import com.gs.collections.impl.block.factory.Functions; +import com.gs.collections.impl.block.function.checked.CheckedFunction; +import com.gs.collections.impl.factory.Lists; +import com.gs.collections.impl.factory.Multimaps; +import com.gs.collections.impl.list.mutable.FastList; + +import kafka.common.ErrorMapping; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.context.SmartLifecycle; +import org.springframework.integration.kafka.core.BrokerAddress; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.ConsumerException; +import org.springframework.integration.kafka.core.FetchRequest; +import org.springframework.integration.kafka.core.KafkaConsumerDefaults; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.KafkaMessageBatch; +import org.springframework.integration.kafka.core.KafkaTemplate; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.Result; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +/** + * @author Marius Bogoevici + */ +public class KafkaMessageListenerContainer implements SmartLifecycle { + + private static final Log log = LogFactory.getLog(KafkaMessageListenerContainer.class); + + public static final Function, Partition> keyFunction = Functions.getKeyFunction(); + + private final GetOffsetForPartitionFunction getOffset = new GetOffsetForPartitionFunction(); + + private final PartitionToLeaderFunction getLeader = new PartitionToLeaderFunction(); + + private final Function passThru = Functions.getPassThru(); + + private final LaunchFetchTaskProcedure launchFetchTask = new LaunchFetchTaskProcedure(); + + private final Object lifecycleMonitor = new Object(); + + private final KafkaTemplate kafkaTemplate; + + private Partition[] partitions; + + private String[] topics; + + public boolean autoStartup = true; + + private Executor fetchTaskExecutor; + + private Executor adminTaskExecutor = Executors.newSingleThreadExecutor(); + + private int concurrency = 1; + + private volatile boolean running = false; + + private int maxFetch = KafkaConsumerDefaults.FETCH_SIZE_INT; + + private int queueSize = 1024; + + private MessageListener messageListener; + + private ErrorHandler errorHandler = new LoggingErrorHandler(); + + private volatile OffsetManager offsetManager; + + private ConcurrentMap fetchOffsets; + + private ConcurrentMessageListenerDispatcher messageDispatcher; + + private final MutableMultimap partitionsByBrokerMap = Multimaps.mutable.set.with(); + + public KafkaMessageListenerContainer(ConnectionFactory connectionFactory, Partition... partitions) { + Assert.notNull(connectionFactory, "A connection factory must be supplied"); + Assert.notEmpty(partitions, "A list of partitions must be provided"); + Assert.noNullElements(partitions, "The list of partitions cannot contain null elements"); + this.kafkaTemplate = new KafkaTemplate(connectionFactory); + this.partitions = partitions; + } + + public KafkaMessageListenerContainer(final ConnectionFactory connectionFactory, String... topics) { + Assert.notNull(connectionFactory, "A connection factory must be supplied"); + Assert.notNull(topics, "A list of topics must be provided"); + Assert.noNullElements(topics, "The list of topics cannot contain null elements"); + this.kafkaTemplate = new KafkaTemplate(connectionFactory); + this.topics = topics; + } + + private static Partition[] getPartitionsForTopics(final ConnectionFactory connectionFactory, String[] topics) { + MutableList partitionList = flatCollect(topics, new GetPartitionsForTopic(connectionFactory)); + return partitionList.toArray(new Partition[partitionList.size()]); + } + + public OffsetManager getOffsetManager() { + return offsetManager; + } + + public void setOffsetManager(OffsetManager offsetManager) { + this.offsetManager = offsetManager; + } + + public MessageListener getMessageListener() { + return messageListener; + } + + public void setMessageListener(MessageListener messageListener) { + this.messageListener = messageListener; + } + + public ErrorHandler getErrorHandler() { + return errorHandler; + } + + public void setErrorHandler(ErrorHandler errorHandler) { + this.errorHandler = errorHandler; + } + + public int getConcurrency() { + return concurrency; + } + + /** + * The maximum number of concurrent {@link MessageListener}s running. Messages from within the same + * partition will be processed sequentially. + * @param concurrency the concurrency maximum number + */ + public void setConcurrency(int concurrency) { + this.concurrency = concurrency; + } + + public Executor getFetchTaskExecutor() { + return fetchTaskExecutor; + } + + /** + * The task executor for fetch operations + * @param fetchTaskExecutor the Executor for fetch operations + */ + public void setFetchTaskExecutor(Executor fetchTaskExecutor) { + this.fetchTaskExecutor = fetchTaskExecutor; + } + + /** + * @return the task executor for leader and offset updates. + */ + public Executor getAdminTaskExecutor() { + return adminTaskExecutor; + } + + public void setAdminTaskExecutor(Executor adminTaskExecutor) { + this.adminTaskExecutor = adminTaskExecutor; + } + + /** + * @return the maximum amount of data (in bytes) that pollers will fetch in one round + */ + public int getMaxFetch() { + return maxFetch; + } + + public int getQueueSize() { + return queueSize; + } + + /** + * The maximum number of messages that are buffered by each concurrent {@link MessageListener} runner. + * Increasing the value may increase throughput, but also increases the memory consumption. + * @param queueSize the queue size + */ + public void setQueueSize(int queueSize) { + this.queueSize = queueSize; + } + + public void setMaxFetch(int maxFetch) { + this.maxFetch = maxFetch; + } + + @Override + public boolean isAutoStartup() { + return autoStartup; + } + + public void setAutoStartup(boolean autoStartup) { + this.autoStartup = autoStartup; + } + + @Override + public void stop(Runnable callback) { + synchronized (lifecycleMonitor) { + if (running) { + this.running = false; + try { + this.offsetManager.flush(); + } + catch (IOException e) { + log.error("Error while flushing:", e); + } + try { + this.offsetManager.close(); + } + catch (IOException e) { + log.error("Error while closing:", e); + } + this.messageDispatcher.stop(); + } + } + if (callback != null) { + callback.run(); + } + } + + @Override + public void start() { + synchronized (lifecycleMonitor) { + if (!running) { + if (partitions == null) { + partitions = getPartitionsForTopics(kafkaTemplate.getConnectionFactory(), topics); + } + this.running = true; + if (this.offsetManager == null) { + this.offsetManager = new MetadataStoreOffsetManager(kafkaTemplate.getConnectionFactory()); + } + // initialize the fetch offset table - defer to OffsetManager for retrieving them + ImmutableList partitionsAsList = Lists.immutable.with(partitions); + this.fetchOffsets = new ConcurrentHashMap(partitionsAsList.toMap(passThru, getOffset)); + this.messageDispatcher = new ConcurrentMessageListenerDispatcher(messageListener, errorHandler, + Arrays.asList(partitions), offsetManager, concurrency, queueSize); + this.messageDispatcher.start(); + partitionsByBrokerMap.putAll(partitionsAsList.groupBy(getLeader)); + if (fetchTaskExecutor == null) { + fetchTaskExecutor = Executors.newFixedThreadPool(partitionsByBrokerMap.size()); + } + partitionsByBrokerMap.forEachKey(launchFetchTask); + } + } + } + + @Override + public void stop() { + this.stop(null); + } + + @Override + public boolean isRunning() { + return this.running; + } + + @Override + public int getPhase() { + return 0; + } + + /** + * Fetches data from Kafka for a group of partitions, located on the same broker. + */ + public class FetchTask implements Runnable { + + private BrokerAddress brokerAddress; + + public FetchTask(BrokerAddress brokerAddress) { + this.brokerAddress = brokerAddress; + } + + @Override + public void run() { + boolean wasInterrupted = false; + while (isRunning()) { + MutableCollection fetchPartitions; + synchronized (partitionsByBrokerMap) { + // retrieve the partitions for the current polling cycle + fetchPartitions = partitionsByBrokerMap.get(brokerAddress); + // do not proceed until there is something to read from + while (isRunning() && CollectionUtils.isEmpty(fetchPartitions)) { + try { + // we only got here because there were no partitions to read from, + // so block until there is a change this prevents FetchTasks + // from busy waiting while leaders or offsets are being refreshed + // TODO: ideally we should use separate monitors for each task + partitionsByBrokerMap.wait(); + // see if the changes affect us + fetchPartitions = partitionsByBrokerMap.get(brokerAddress); + } + catch (InterruptedException e) { + wasInterrupted = true; + } + } + } + // we've just exited a potentially blocking operation. Is the component still running? + if (isRunning()) { + Set partitionsWithRemainingData; + boolean hasErrors; + do { + partitionsWithRemainingData = new HashSet(); + hasErrors = false; + try { + MutableCollection fetchRequests = + fetchPartitions.collect(new PartitionToFetchRequestFunction()); + Result result = kafkaTemplate.receive(fetchRequests); + // process successful messages first + Iterable batches = result.getResults().values(); + for (KafkaMessageBatch batch : batches) { + if (!batch.getMessages().isEmpty()) { + long highestFetchedOffset = 0; + for (KafkaMessage kafkaMessage : batch.getMessages()) { + // fetch operations may return entire blocks of compressed messages, + // which may have lower offsets than the ones requested + // thus a batch may contain messages that have been processed already + if (kafkaMessage.getMetadata().getOffset() >= fetchOffsets.get(batch.getPartition())) { + messageDispatcher.dispatch(kafkaMessage); + } + highestFetchedOffset = + Math.max(highestFetchedOffset, kafkaMessage.getMetadata().getNextOffset()); + } + fetchOffsets.replace(batch.getPartition(), highestFetchedOffset); + // if there are still messages on server, we can go on and retrieve more + if (highestFetchedOffset < batch.getHighWatermark()) { + partitionsWithRemainingData.add(batch.getPartition()); + } + } + } + // handle errors + if (result.getErrors().size() > 0) { + hasErrors = true; + + // find partitions with leader errors and + PartitionIterable> partitionByLeaderErrors = + partition(result.getErrors().entrySet(), new IsLeaderErrorPredicate()); + RichIterable partitionsWithLeaderErrors = + partitionByLeaderErrors.getSelected().collect(keyFunction); + resetLeaders(partitionsWithLeaderErrors); + + PartitionIterable> partitionsWithOffsetsOutOfRange = + partitionByLeaderErrors.getRejected() + .partition(new IsOffsetOutOfRangePredicate()); + resetOffsets(partitionsWithOffsetsOutOfRange.getSelected() + .collect(keyFunction) + .toSet()); + // it's not a leader issue + stopFetchingFromPartitions(partitionsWithOffsetsOutOfRange.getRejected() + .collect(keyFunction)); + } + } + catch (ConsumerException e) { + // this is a broker error, and we cannot recover from it. + // Reset leaders and stop fetching data from this broker altogether + log.error(e); + resetLeaders(fetchPartitions.toImmutable()); + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + return; + } + } while (!hasErrors && !partitionsWithRemainingData.isEmpty()); + } + } + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + } + + + private void resetLeaders(final Iterable partitionsToReset) { + stopFetchingFromPartitions(partitionsToReset); + adminTaskExecutor.execute(new UpdateLeadersTask(partitionsToReset)); + } + + + private void resetOffsets(final Collection partitionsToResetOffsets) { + stopFetchingFromPartitions(partitionsToResetOffsets); + adminTaskExecutor.execute(new UpdateOffsetsTask(partitionsToResetOffsets)); + } + + private void stopFetchingFromPartitions(Iterable partitions) { + synchronized (partitionsByBrokerMap) { + for (Partition partition : partitions) { + partitionsByBrokerMap.remove(brokerAddress, partition); + } + } + } + + private class UpdateLeadersTask implements Runnable { + private final Iterable partitionsToReset; + + public UpdateLeadersTask(Iterable partitionsToReset) { + this.partitionsToReset = partitionsToReset; + } + + @Override + public void run() { + FastList partitionsAsList = FastList.newList(partitionsToReset); + FastList topics = partitionsAsList.collect(new PartitionToTopicFunction()).distinct(); + kafkaTemplate.getConnectionFactory().refreshLeaders(topics); + Map leaders = kafkaTemplate.getConnectionFactory().getLeaders(partitionsToReset); + synchronized (partitionsByBrokerMap) { + forEachKeyValue(leaders, new AddPartitionToBrokerProcedure()); + partitionsByBrokerMap.notifyAll(); + } + } + + } + + private class UpdateOffsetsTask implements Runnable { + + private final Collection partitionsToResetOffsets; + + public UpdateOffsetsTask(Collection partitionsToResetOffsets) { + this.partitionsToResetOffsets = partitionsToResetOffsets; + } + + @Override + public void run() { + offsetManager.resetOffsets(partitionsToResetOffsets); + for (Partition partition : partitionsToResetOffsets) { + fetchOffsets.replace(partition, offsetManager.getOffset(partition)); + } + synchronized (partitionsByBrokerMap) { + for (Partition partitionsToResetOffset : partitionsToResetOffsets) { + partitionsByBrokerMap.put(brokerAddress, partitionsToResetOffset); + } + // notify any waiting task that the partition allocation has changed + partitionsByBrokerMap.notifyAll(); + } + } + + } + + @SuppressWarnings("serial") + private class IsLeaderErrorPredicate implements Predicate> { + + @Override + public boolean accept(Map.Entry each) { + return each.getValue() == ErrorMapping.NotLeaderForPartitionCode() + || each.getValue() == ErrorMapping.UnknownTopicOrPartitionCode(); + } + + } + + @SuppressWarnings("serial") + private class IsOffsetOutOfRangePredicate implements Predicate> { + + @Override + public boolean accept(Map.Entry each) { + return each.getValue() == ErrorMapping.OffsetOutOfRangeCode(); + } + + } + } + + @SuppressWarnings("serial") + class GetOffsetForPartitionFunction extends CheckedFunction { + + @Override + public Long safeValueOf(Partition object) throws Exception { + try { + return offsetManager.getOffset(object); + } + catch (Exception e) { + log.error(e); + throw e; + } + } + + } + + @SuppressWarnings("serial") + private class PartitionToLeaderFunction implements Function { + + @Override + public BrokerAddress valueOf(Partition partition) { + return kafkaTemplate.getConnectionFactory().getLeader(partition); + } + + } + + @SuppressWarnings("serial") + private class LaunchFetchTaskProcedure implements Procedure { + + @Override + public void value(BrokerAddress brokerAddress) { + fetchTaskExecutor.execute(new FetchTask(brokerAddress)); + } + + } + + @SuppressWarnings("serial") + private class PartitionToFetchRequestFunction implements Function { + + @Override + public FetchRequest valueOf(Partition partition) { + return new FetchRequest(partition, fetchOffsets.get(partition), maxFetch); + } + + } + + @SuppressWarnings("serial") + static class GetPartitionsForTopic extends CheckedFunction> { + + private final ConnectionFactory connectionFactory; + + public GetPartitionsForTopic(ConnectionFactory connectionFactory) { + this.connectionFactory = connectionFactory; + } + + @Override + public Iterable safeValueOf(String topic) throws Exception { + return connectionFactory.getPartitions(topic); + } + + } + + @SuppressWarnings("serial") + private class PartitionToTopicFunction implements Function { + + @Override + public String valueOf(Partition object) { + return object.getTopic(); + } + + } + + @SuppressWarnings("serial") + private class AddPartitionToBrokerProcedure implements Procedure2 { + + @Override + public void value(Partition partition, BrokerAddress newBrokerAddress) { + partitionsByBrokerMap.put(newBrokerAddress, partition); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LoggingErrorHandler.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LoggingErrorHandler.java new file mode 100644 index 0000000000..c149a354fb --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LoggingErrorHandler.java @@ -0,0 +1,38 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.util.ObjectUtils; + +/** + * @author Marius Bogoevici + */ +public class LoggingErrorHandler implements ErrorHandler { + + private static final Log log = LogFactory.getLog(LoggingErrorHandler.class); + + @Override + public void handle(Exception thrownException, KafkaMessage message) { + log.error("Error while processing: " + ObjectUtils.nullSafeToString(message), thrownException); + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MessageListener.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MessageListener.java new file mode 100644 index 0000000000..554df9efef --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MessageListener.java @@ -0,0 +1,35 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.springframework.integration.kafka.core.KafkaMessage; + +/** + * Listener for handling incoming Kafka messages + * + * @author Marius Bogoevici + */ +public interface MessageListener { + + /** + * Executes when a Kafka message is received + * @param message the Kafka message to be processed + */ + void onMessage(KafkaMessage message); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MetadataStoreOffsetManager.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MetadataStoreOffsetManager.java new file mode 100644 index 0000000000..92a3611ff2 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MetadataStoreOffsetManager.java @@ -0,0 +1,188 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.listener; + +import java.io.Closeable; +import java.io.Flushable; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import kafka.common.ErrorMapping; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.integration.kafka.core.BrokerAddress; +import org.springframework.integration.kafka.core.Connection; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.ConsumerException; +import org.springframework.integration.kafka.core.KafkaConsumerDefaults; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.Result; +import org.springframework.integration.metadata.MetadataStore; +import org.springframework.integration.metadata.SimpleMetadataStore; + +/** + * An {@link OffsetManager} that persists offsets into a {@link MetadataStore}. + * + * @author Marius Bogoevici + */ +public class MetadataStoreOffsetManager implements OffsetManager { + + private final static Log log = LogFactory.getLog(MetadataStoreOffsetManager.class); + + private String consumerId = KafkaConsumerDefaults.GROUP_ID; + + private MetadataStore metadataStore = new SimpleMetadataStore(); + + private ConnectionFactory connectionFactory; + + private Map initialOffsets; + + private long referenceTimestamp = KafkaConsumerDefaults.DEFAULT_OFFSET_RESET; + + public MetadataStoreOffsetManager(ConnectionFactory connectionFactory) { + this(connectionFactory, null); + } + + public MetadataStoreOffsetManager(ConnectionFactory connectionFactory, Map initialOffsets) { + this.connectionFactory = connectionFactory; + this.initialOffsets = initialOffsets == null? new HashMap() : initialOffsets; + } + + public String getConsumerId() { + return consumerId; + } + + /** + * The identifier of a consumer of Kafka messages. Allows to store separate sets of offsets in the + * {@link MetadataStore}. + * @param consumerId the consumer ID + */ + public void setConsumerId(String consumerId) { + this.consumerId = consumerId; + } + + public MetadataStore getMetadataStore() { + return metadataStore; + } + + /** + * The backing {@link MetadataStore} for storing offsets. + * @param metadataStore a fully configured {@link MetadataStore} instance + */ + public void setMetadataStore(MetadataStore metadataStore) { + this.metadataStore = metadataStore; + } + + public long getReferenceTimestamp() { + return referenceTimestamp; + } + + /** + * A timestamp to be used for resetting initial offsets, if they are not available in the {@link MetadataStore} + * @param referenceTimestamp the reset timestamp for initial offsets + */ + public void setReferenceTimestamp(long referenceTimestamp) { + this.referenceTimestamp = referenceTimestamp; + } + + /** + * @see OffsetManager#updateOffset(Partition, long) + */ + @Override + public synchronized void updateOffset(Partition partition, long offset) { + metadataStore.put(asKey(partition), Long.toString(offset)); + } + + /** + * @see OffsetManager#getOffset(Partition) + */ + @Override + public synchronized long getOffset(Partition partition) { + Long offsetInMetadataStore = getOffsetFromMetadataStore(partition); + if (offsetInMetadataStore == null) { + if (this.initialOffsets.containsKey(partition)) { + return this.initialOffsets.get(partition); + } + else { + BrokerAddress leader = this.connectionFactory.getLeader(partition); + if (leader == null) { + throw new ConsumerException("No leader found for " + partition.toString()); + } + Connection connection = this.connectionFactory.connect(leader); + Result offsetResult = connection.fetchInitialOffset(referenceTimestamp, partition); + if (offsetResult.getErrors().size() > 0) { + throw new ConsumerException(ErrorMapping.exceptionFor(offsetResult.getError(partition))); + } + if (!offsetResult.getResults().containsKey(partition)) { + throw new IllegalStateException("Result does not contain an expected value"); + } + return offsetResult.getResult(partition); + } + } + else { + return offsetInMetadataStore; + } + } + + @Override + public synchronized void resetOffsets(Collection partitionsToReset) { + // any information about those offsets is invalid and can be ignored + for (Partition partition : partitionsToReset) { + metadataStore.remove(asKey(partition)); + initialOffsets.remove(partition); + } + } + + @Override + public void close() throws IOException { + // Flush before closing. This may be redundant, but ensures that the metadata store is closed properly + flush(); + if (metadataStore instanceof Closeable) { + ((Closeable) metadataStore).close(); + } + } + + @Override + public void flush() throws IOException { + if (metadataStore instanceof Flushable) { + ((Flushable) metadataStore).flush(); + } + } + + + private Long getOffsetFromMetadataStore(Partition partition) { + String storedOffsetValueAsString = this.metadataStore.get(asKey(partition)); + Long storedOffsetValue = null; + if (storedOffsetValueAsString != null) { + try { + storedOffsetValue = Long.parseLong(storedOffsetValueAsString); + } + catch (NumberFormatException e) { + log.warn("Invalid value: " + storedOffsetValueAsString); + } + } + return storedOffsetValue; + } + + private String asKey(Partition partition) { + return partition.getTopic() + ":" + partition.getId() + ":" + consumerId; + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/OffsetManager.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/OffsetManager.java new file mode 100644 index 0000000000..b41207bcfc --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/OffsetManager.java @@ -0,0 +1,54 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.listener; + +import java.io.Closeable; +import java.io.Flushable; +import java.util.Collection; + +import org.springframework.integration.kafka.core.Partition; + +/** + * Stores and retrieves offsets for a Kafka consumer + * + * @author Marius Bogoevici + */ +public interface OffsetManager extends Closeable, Flushable { + + /** + * Updates the offset for a given {@link Partition} + * @param partition the partition whose offset is to be updated + * @param offset the new offset value + */ + void updateOffset(Partition partition, long offset); + + /** + * Retrieves the offset for a given {@link Partition} + * + * @param partition the partition to be + * @return the offset value + */ + long getOffset(Partition partition); + + /** + * Resets offsets for the given {@link Partition}s. To be invoked when the values stored are invalid, + * so a client cannot resume from that position. Implementations must decide on the best strategy to follow. + * @param partition to reset + */ + void resetOffsets(Collection partition); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/QueueingMessageListenerInvoker.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/QueueingMessageListenerInvoker.java new file mode 100644 index 0000000000..8053b76f8d --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/QueueingMessageListenerInvoker.java @@ -0,0 +1,126 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.listener; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +import org.springframework.context.Lifecycle; +import org.springframework.integration.kafka.core.KafkaMessage; + +/** + * Invokes a delegate {@link MessageListener} for all the messages passed to it, storing them + * in an internal queue. + * + * @author Marius Bogoevici + */ +class QueueingMessageListenerInvoker implements Runnable, Lifecycle { + + private BlockingQueue messages; + + private volatile boolean running = false; + + private final MessageListener delegate; + + private final OffsetManager offsetManager; + + private final ErrorHandler errorHandler; + + public QueueingMessageListenerInvoker(int capacity, OffsetManager offsetManager, MessageListener delegate, + ErrorHandler errorHandler) { + this.offsetManager = offsetManager; + this.delegate = delegate; + this.errorHandler = errorHandler; + this.messages = new ArrayBlockingQueue(capacity, true); + } + + /** + * Add a message to the queue, blocking if the queue has reached its maximum capacity. + * Interrupts will be ignored for as long as the component's {@code running} flag is set to true, but will + * be deferred for when the method returns. + * @param message the KafkaMessage to add + */ + public void enqueue(KafkaMessage message) { + boolean wasInterruptedWhileRunning = false; + if (this.running) { + boolean added = false; + // handle the case when the thread is interrupted while the adapter is still running + // retry adding the message to the queue until either we succeed, or the adapter is stopped + while (!added && this.running) { + try { + this.messages.put(message); + added = true; + } + catch (InterruptedException e) { + // we ignore the interruption signal if we are still running, but pass it on if we are stopped + wasInterruptedWhileRunning = true; + } + } + } + if (wasInterruptedWhileRunning) { + Thread.currentThread().interrupt(); + } + } + + @Override + public void start() { + this.running = true; + } + + @Override + public void stop() { + this.running = false; + } + + @Override + public boolean isRunning() { + return this.running; + } + + /** + * Runs uninterruptibly as long as {@code running} is true, but if interrupted, will defer + * propagating the interruption flag at the end. + */ + @Override + public void run() { + boolean wasInterrupted = false; + while (this.running) { + try { + KafkaMessage message = messages.take(); + try { + delegate.onMessage(message); + } + catch (Exception e) { + if (errorHandler != null) { + errorHandler.handle(e, message); + } + } + finally { + offsetManager.updateOffset(message.getMetadata().getPartition(), + message.getMetadata().getNextOffset()); + } + } + catch (InterruptedException e) { + wasInterrupted = true; + } + } + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/package-info.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/package-info.java new file mode 100644 index 0000000000..c2a49222f5 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/package-info.java @@ -0,0 +1,4 @@ +/** + * Provides message listener container support + */ +package org.springframework.integration.kafka.listener; diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaHeaders.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaHeaders.java index 148235812b..eec5496cfe 100644 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaHeaders.java +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaHeaders.java @@ -28,4 +28,8 @@ public abstract class KafkaHeaders { public static final String MESSAGE_KEY = PREFIX + "messageKey"; + public static final String PARTITION_ID = PREFIX + "_partitionId"; + + public static final String OFFSET = PREFIX + "_offset"; + } diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/MessageUtils.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/MessageUtils.java new file mode 100644 index 0000000000..b50db52c19 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/MessageUtils.java @@ -0,0 +1,43 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.util; + +import kafka.serializer.Decoder; +import kafka.utils.Utils$; + +import org.springframework.integration.kafka.core.KafkaMessage; + +/** + * @author Marius Bogoevici + */ +public class MessageUtils { + + public static T decodeKey(KafkaMessage message, Decoder decoder) { + if (message.getMessage().isNull() || !message.getMessage().hasKey()) { + return null; + } + return decoder.fromBytes(Utils$.MODULE$.readBytes(message.getMessage().key())); + } + + public static T decodePayload(KafkaMessage message, Decoder decoder) { + if (message.getMessage().isNull()) { + return null; + } + return decoder.fromBytes(Utils$.MODULE$.readBytes(message.getMessage().payload())); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractBrokerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractBrokerTests.java new file mode 100644 index 0000000000..dcfa537e1f --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractBrokerTests.java @@ -0,0 +1,157 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static scala.collection.JavaConversions.asScalaBuffer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import com.gs.collections.api.RichIterable; +import com.gs.collections.api.block.function.Function2; +import com.gs.collections.api.multimap.Multimap; +import com.gs.collections.api.multimap.MutableMultimap; +import com.gs.collections.api.tuple.Pair; +import com.gs.collections.impl.factory.Multimaps; +import com.gs.collections.impl.tuple.Tuples; +import kafka.admin.AdminUtils; +import kafka.producer.KeyedMessage; +import kafka.producer.Producer; +import kafka.producer.ProducerConfig; +import kafka.serializer.StringEncoder; +import kafka.utils.TestUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.After; +import scala.collection.JavaConversions; +import scala.collection.Map; +import scala.collection.immutable.List$; +import scala.collection.immutable.Map$; +import scala.collection.immutable.Seq; + +import org.springframework.integration.kafka.core.Configuration; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.DefaultConnectionFactory; +import org.springframework.integration.kafka.core.BrokerAddressListConfiguration; +import org.springframework.integration.kafka.rule.KafkaRule; + +/** + * @author Marius Bogoevici + */ +public abstract class AbstractBrokerTests { + + private static final Log log = LogFactory.getLog(AbstractBrokerTests.class); + + public static final String TEST_TOPIC = "test-topic"; + + public abstract KafkaRule getKafkaRule(); + + @After + public void cleanUp() { + deleteTopic(TEST_TOPIC); + } + + @SuppressWarnings("unchecked") + public void createTopic(String topicName, int partitionCount, int brokers, int replication) { + MutableMultimap partitionDistribution = createPartitionDistribution(partitionCount, brokers, replication); + AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK(getKafkaRule().getZkClient(), + topicName, toKafkaPartitionMap(partitionDistribution), + AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK$default$4(), + AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK$default$5()); + if (getKafkaRule().isEmbedded()) { + for (int i = 0; i < partitionDistribution.keysView().size(); i++) { + TestUtils.waitUntilMetadataIsPropagated(asScalaBuffer(getKafkaRule().getKafkaServers()), topicName, i, 5000L); + } + } else { + sleep(partitionCount * 200); + } + } + + public void deleteTopic(String topicName) { + AdminUtils.deleteTopic(getKafkaRule().getZkClient(), topicName); + if (getKafkaRule().isEmbedded()) { + TestUtils.waitUntilMetadataIsPropagated(asScalaBuffer(getKafkaRule().getKafkaServers()), topicName, 0, 5000L); + } else { + sleep(1000); + } + } + + public MutableMultimap createPartitionDistribution(int partitionCount, int brokers, int replication) { + MutableMultimap partitionDistribution = Multimaps.mutable.list.with(); + for (int i = 0; i < partitionCount; i++) { + for (int j = 0; j < replication; j++) { + partitionDistribution.put(i, (i + j) % brokers); + } + } + return partitionDistribution; + } + + + public Configuration getKafkaConfiguration() { + return new BrokerAddressListConfiguration(getKafkaRule().getBrokerAddresses()); + } + + public static scala.collection.Seq> createMessages(int count, String topic) { + return createMessagesInRange(0,count-1,topic); + } + + public static scala.collection.Seq> createMessagesInRange(int start, int end, String topic) { + List> messages = new ArrayList>(); + for (int i=start; i<= end; i++) { + messages.add(new KeyedMessage(topic, "Key " + i, i, "Message " + i)); + } + return asScalaBuffer(messages).toSeq(); + } + + public Producer createStringProducer(int compression) { + Properties producerConfig = TestUtils.getProducerConfig(getKafkaRule().getBrokersAsString(), + TestPartitioner.class.getCanonicalName()); + producerConfig.put("serializer.class", StringEncoder.class.getCanonicalName()); + producerConfig.put("key.serializer.class", StringEncoder.class.getCanonicalName()); + producerConfig.put("compression.codec", Integer.toString(compression)); + return new Producer(new ProducerConfig(producerConfig)); + } + + public ConnectionFactory getKafkaBrokerConnectionFactory() throws Exception { + DefaultConnectionFactory connectionFactory = new DefaultConnectionFactory(getKafkaConfiguration()); + connectionFactory.afterPropertiesSet(); + return connectionFactory; + } + + @SuppressWarnings({"rawtypes", "serial", "deprecation"}) + private Map toKafkaPartitionMap(Multimap partitions) { + java.util.Map> m = partitions.toMap().collect(new Function2, Pair>>() { + @Override + public Pair> value(Integer argument1, RichIterable argument2) { + return Tuples.pair((Object) argument1, List$.MODULE$.fromArray(argument2.toArray(new Object[0])).toSeq()); + } + }); + return Map$.MODULE$.apply(JavaConversions.asScalaMap(m).toSeq()); + } + + private static void sleep(int time) { + try { + Thread.sleep(time); + } + catch (InterruptedException e) { + log.error(e); + } + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractMessageListenerContainerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractMessageListenerContainerTests.java new file mode 100644 index 0000000000..07a9695d0e --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractMessageListenerContainerTests.java @@ -0,0 +1,224 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.collection.IsEmptyCollection.empty; +import static org.hamcrest.core.IsCollectionContaining.hasItem; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; +import static org.springframework.integration.kafka.util.MessageUtils.decodeKey; +import static org.springframework.integration.kafka.util.MessageUtils.decodePayload; + +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.gs.collections.api.RichIterable; +import com.gs.collections.api.bag.MutableBag; +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.block.function.Function2; +import com.gs.collections.api.block.procedure.Procedure; +import com.gs.collections.api.list.MutableList; +import com.gs.collections.api.map.MutableMap; +import com.gs.collections.api.multimap.list.MutableListMultimap; +import com.gs.collections.api.set.MutableSet; +import com.gs.collections.api.tuple.Pair; +import com.gs.collections.impl.factory.Sets; +import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; +import com.gs.collections.impl.tuple.Tuples; +import kafka.serializer.StringDecoder; +import kafka.utils.VerifiableProperties; +import org.hamcrest.Matchers; + +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.KafkaMessage; + +/** + * @author Marius Bogoevici + */ +public abstract class AbstractMessageListenerContainerTests extends AbstractBrokerTests { + + public void runMessageListenerTest(int maxReceiveSize, int concurrency, int partitionCount, int testMessageCount, int divisionFactor, int compressionCodec, String topic) throws Exception { + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + for (int i = 0; i < partitionCount; i++) { + if(i % divisionFactor == 0) { + readPartitions.add(new Partition(topic, i)); + } + } + final KafkaMessageListenerContainer kafkaMessageListenerContainer = new KafkaMessageListenerContainer(connectionFactory, readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(maxReceiveSize); + kafkaMessageListenerContainer.setConcurrency(concurrency); + + int expectedMessageCount = testMessageCount / divisionFactor; + + final MutableListMultimap receivedData = new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + kafkaMessageListenerContainer.setMessageListener(new MessageListener() { + @Override + public void onMessage(KafkaMessage message) { + StringDecoder decoder = new StringDecoder(new VerifiableProperties()); + receivedData.put(message.getMetadata().getPartition().getId(),new KeyedMessageWithOffset(decodeKey(message, decoder), decodePayload(message, decoder), message.getMetadata().getOffset(), Thread.currentThread().getName(), message.getMetadata().getPartition().getId())); + latch.countDown(); + } + }); + + kafkaMessageListenerContainer.start(); + + createStringProducer(compressionCodec).send(createMessages(testMessageCount, topic)); + + latch.await((expectedMessageCount/5000) + 1, TimeUnit.MINUTES); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, concurrency, partitionCount, testMessageCount, expectedMessageCount, readPartitions, divisionFactor); + + } + + @SuppressWarnings("serial") + public void validateMessageReceipt(MutableListMultimap receivedData, int concurrency, int partitionCount, int testMessageCount, int expectedMessageCount, ArrayList readPartitions, int divisionFactor) { + // Group messages received by processing thread + MutableListMultimap messagesByThread = receivedData.valuesView().toList().groupBy(new Function() { + @Override + public String valueOf(KeyedMessageWithOffset object) { + return object.getThreadName(); + } + }); + + // Execution has taken place on as many distinct threads as configured + assertThat(messagesByThread.keysView().size(), Matchers.equalTo(concurrency)); + + // Group partitions by thread + MutableMap> partitionsByThread = messagesByThread.toMap().collect(new Function2, Pair>>() { + @Override + public Pair> value(String argument1, RichIterable argument2) { + return Tuples.pair(argument1, argument2.collect(new Function() { + @Override + public Integer valueOf(KeyedMessageWithOffset object) { + return object.getPartition(); + } + }).toSet()); + } + }); + + // Messages from a partition have been executed on the same thread and groups are mutually exclusive + final MutableSet validatedPartitions = Sets.mutable.of(); + partitionsByThread.valuesView().forEach(new Procedure>() { + @Override + public void value(MutableSet partitions) { + assertThat(validatedPartitions.intersect(partitions), empty()); + validatedPartitions.addAll(partitions); + } + }); + + // All partitions are accounted for, but only the ones that we were expecting to read from + for (int i = 0; i < partitionCount; i++) { + if (i % divisionFactor == 0) { + assertThat(validatedPartitions, hasItem(i)); + } else { + assertThat(validatedPartitions, not(hasItem(i))); + } + } + + // Sort data by payload in order to identify duplicates + MutableList sortedPayloads = receivedData.valuesView().toList().collect(new Function() { + @Override + public String valueOf(KeyedMessageWithOffset object) { + return object.getPayload(); + } + }).sortThis(); + + // Remove unique values - what is left are duplicates + MutableBag duplicates = sortedPayloads.toBag(); + duplicates.removeAll(sortedPayloads.toSet()); + + // The final set has exactly the same size as the message count + assertThat(sortedPayloads, hasSize(expectedMessageCount)); + + // There are no duplicates - all messages have been received only once + assertThat(duplicates, hasSize(0)); + + // Group offsets by partition + MutableMap> offsetsByPartition = receivedData.toMap().collect(new Function2, Pair>>() { + @Override + public Pair> value(Integer partition, RichIterable argument2) { + return Tuples.pair(partition, argument2.collect(new Function() { + @Override + public Long valueOf(KeyedMessageWithOffset object) { + return object.getOffset(); + } + }).toList()); + } + }); + + // Check that the sequence of offsets has been processed in ascending order, with no gaps + for (MutableList offsetsForPartition : offsetsByPartition.valuesView()) { + for (int i = 0; i < offsetsForPartition.size() - 1; i++) { + assertThat(offsetsForPartition.get(i+1), equalTo(offsetsForPartition.get(i) + 1)); + } + } + } + + static class KeyedMessageWithOffset { + + String key; + + String payload; + + Long offset; + + String threadName; + + int partition; + + public KeyedMessageWithOffset(String key, String payload, Long offset, String threadName, int partition) { + this.key = key; + this.payload = payload; + this.offset = offset; + this.threadName = threadName; + this.partition = partition; + } + + public String getKey() { + return key; + } + + public String getPayload() { + return payload; + } + + public Long getOffset() { + return offset; + } + + public String getThreadName() { + return threadName; + } + + public int getPartition() { + return partition; + } + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests-context.xml b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests-context.xml new file mode 100644 index 0000000000..d1ddd66a41 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests-context.xml @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests.java new file mode 100644 index 0000000000..709436c5fb --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests.java @@ -0,0 +1,71 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.Matchers.notNullValue; + + +import kafka.message.NoCompressionCodec$; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.context.support.ClassPathXmlApplicationContext; +import org.springframework.integration.channel.QueueChannel; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.rule.KafkaRule; +import org.springframework.messaging.Message; + +/** + * @author Marius Bogoevici + */ +public class ChannelAdapterWithXmlConfigurationTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaRule kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaRule getKafkaRule() { + return this.kafkaEmbeddedBrokerRule; + } + + @Test + public void testConsumptionWithXmlConfiguration() throws Exception { + + System.setProperty("kafka.test.port", + String.valueOf(kafkaEmbeddedBrokerRule.getBrokerAddresses().get(0).getPort())); + System.setProperty("kafka.test.topic", TEST_TOPIC); + + createTopic(TEST_TOPIC, 1, 1, 1); + + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessages(100, TEST_TOPIC)); + + ClassPathXmlApplicationContext context = + new ClassPathXmlApplicationContext("ChannelAdapterWithXmlConfigurationTests-context.xml", + ChannelAdapterWithXmlConfigurationTests.class); + + QueueChannel output = context.getBean("output", QueueChannel.class); + + for (int i = 0; i < 100; i++) { + Message received = output.receive(1000); + Assert.assertThat(received, notNullValue()); + } + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelSendingMessageListener.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelSendingMessageListener.java new file mode 100644 index 0000000000..9eda772104 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelSendingMessageListener.java @@ -0,0 +1,61 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.listener.MessageListener; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.integration.support.channel.BeanFactoryChannelResolver; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.core.DestinationResolver; + +/** + * @author Marius Bogoevici + */ +public class ChannelSendingMessageListener implements MessageListener, ApplicationContextAware { + + private String channelName; + + private MessageChannel messageChannel; + + public String getChannelName() { + return channelName; + } + + public void setChannelName(String channelName) { + this.channelName = channelName; + } + + DestinationResolver destinationResolver; + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.destinationResolver = new BeanFactoryChannelResolver(applicationContext); + messageChannel = destinationResolver.resolveDestination(channelName); + } + + @Override + public void onMessage(KafkaMessage message) { + byte b[] = new byte[message.getMessage().payloadSize()]; + message.getMessage().payload().get(b); + messageChannel.send(MessageBuilder.withPayload(new String(b)).build()); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionFactoryTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionFactoryTests.java new file mode 100644 index 0000000000..5ec2b7a2be --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionFactoryTests.java @@ -0,0 +1,87 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.collection.IsEmptyCollection.empty; +import static org.junit.Assert.assertThat; + +import java.util.List; + +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.core.BrokerAddress; +import org.springframework.integration.kafka.core.Connection; +import org.springframework.integration.kafka.core.DefaultConnectionFactory; +import org.springframework.integration.kafka.core.BrokerAddressListConfiguration; +import org.springframework.integration.kafka.core.Result; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.ZookeeperConfiguration; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.support.ZookeeperConnect; + +/** + * @author Marius Bogoevici + */ +public class DefaultConnectionFactoryTests extends AbstractBrokerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testCreateConnectionFactoryWithBrokerList() throws Exception { + + createTopic(TEST_TOPIC, 1, 1, 1); + + List brokerAddresses = getKafkaRule().getBrokerAddresses(); + Partition partition = new Partition(TEST_TOPIC, 0); + DefaultConnectionFactory connectionFactory = new DefaultConnectionFactory(new BrokerAddressListConfiguration(brokerAddresses)); + connectionFactory.afterPropertiesSet(); + Connection connection = connectionFactory.connect(getKafkaRule().getBrokerAddresses().get(0)); + Result leaders = connection.findLeaders(TEST_TOPIC); + assertThat(leaders.getErrors().entrySet(), empty()); + assertThat(leaders.getResults().entrySet(), hasSize(1)); + assertThat(leaders.getResults().get(partition), equalTo(getKafkaRule().getBrokerAddresses().get(0))); + } + + @Test + public void testCreateConnectionFactoryWithZookeeper() throws Exception { + + createTopic(TEST_TOPIC, 1, 1, 1); + + + Partition partition = new Partition(TEST_TOPIC, 0); + ZookeeperConnect zookeeperConnect = new ZookeeperConnect(); + zookeeperConnect.setZkConnect(kafkaEmbeddedBrokerRule.getZookeeperConnectionString()); + DefaultConnectionFactory connectionFactory = + new DefaultConnectionFactory(new ZookeeperConfiguration(zookeeperConnect)); + connectionFactory.afterPropertiesSet(); + Connection connection = connectionFactory.connect(getKafkaRule().getBrokerAddresses().get(0)); + Result leaders = connection.findLeaders(TEST_TOPIC); + assertThat(leaders.getErrors().entrySet(), empty()); + assertThat(leaders.getResults().entrySet(), hasSize(1)); + assertThat(leaders.getResults().get(partition), equalTo(getKafkaRule().getBrokerAddresses().get(0))); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionTests.java new file mode 100644 index 0000000000..51e06fcbe1 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionTests.java @@ -0,0 +1,126 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import kafka.producer.Producer; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.core.Connection; +import org.springframework.integration.kafka.core.DefaultConnection; +import org.springframework.integration.kafka.core.FetchRequest; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.KafkaMessageBatch; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.Result; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.serializer.common.StringDecoder; +import org.springframework.integration.kafka.util.MessageUtils; + +/** + * @author Marius Bogoevici + */ +public class DefaultConnectionTests extends AbstractBrokerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testFetchPartitionMetadata() throws Exception { + createTopic(TEST_TOPIC, 1, 1, 1); + Connection brokerConnection = new DefaultConnection(getKafkaRule().getBrokerAddresses().get(0), "client", 64*1024, 3000, 1, 100); + Partition partition = new Partition(TEST_TOPIC, 0); + Result result = brokerConnection.fetchInitialOffset(-1, partition); + Assert.assertEquals(0, result.getErrors().size()); + Assert.assertEquals(1, result.getResults().size()); + Assert.assertEquals(Long.valueOf(0), result.getResults().get(partition)); + } + + @Test + public void testReceiveMessages() throws Exception { + createTopic(TEST_TOPIC, 1, 1, 1); + Producer producer = createStringProducer(0); + producer.send( createMessages(10, TEST_TOPIC)); + Connection brokerConnection = new DefaultConnection(getKafkaRule().getBrokerAddresses().get(0), "client", 64*1024, 3000, 1, 100); + Partition partition = new Partition(TEST_TOPIC, 0); + FetchRequest fetchRequest = new FetchRequest(partition, 0L, 1000); + Result result = brokerConnection.fetch(fetchRequest); + Assert.assertEquals(0, result.getErrors().size()); + Assert.assertEquals(1, result.getResults().size()); + Assert.assertEquals(10, result.getResults().get(partition).getMessages().size()); + Assert.assertEquals(10,result.getResults().get(partition).getHighWatermark()); + StringDecoder decoder = new StringDecoder(); + int i = 0; + for (KafkaMessage kafkaMessage : result.getResults().get(partition).getMessages()) { + Assert.assertEquals("Key " + i, MessageUtils.decodeKey(kafkaMessage, decoder)); + Assert.assertEquals("Message " + i, MessageUtils.decodePayload(kafkaMessage, decoder)); + i++; + } + } + + @Test + public void testReceiveMessagesWithCompression1() throws Exception { + createTopic(TEST_TOPIC, 1, 1, 1); + Producer producer = createStringProducer(1); + producer.send( createMessages(10, TEST_TOPIC)); + Connection brokerConnection = new DefaultConnection(getKafkaRule().getBrokerAddresses().get(0), "client", 64*1024, 3000, 1, 100); + Partition partition = new Partition(TEST_TOPIC, 0); + FetchRequest fetchRequest = new FetchRequest(partition, 0L, 1000); + Result result = brokerConnection.fetch(fetchRequest); + Assert.assertEquals(0, result.getErrors().size()); + Assert.assertEquals(1, result.getResults().size()); + Assert.assertEquals(10, result.getResults().get(partition).getMessages().size()); + Assert.assertEquals(10,result.getResults().get(partition).getHighWatermark()); + StringDecoder decoder = new StringDecoder(); + int i = 0; + for (KafkaMessage kafkaMessage : result.getResults().get(partition).getMessages()) { + Assert.assertEquals("Key " + i, MessageUtils.decodeKey(kafkaMessage, decoder)); + Assert.assertEquals("Message " + i, MessageUtils.decodePayload(kafkaMessage, decoder)); + i++; + } + } + + @Test + public void testReceiveMessagesWithCompression2() throws Exception { + createTopic(TEST_TOPIC, 1, 1, 1); + Producer producer = createStringProducer(2); + producer.send( createMessages(10, TEST_TOPIC)); + Connection brokerConnection = new DefaultConnection(getKafkaRule().getBrokerAddresses().get(0), "client", 64*1024, 3000, 1, 100); + Partition partition = new Partition(TEST_TOPIC, 0); + FetchRequest fetchRequest = new FetchRequest(partition, 0L, 1000); + Result result = brokerConnection.fetch(fetchRequest); + Assert.assertEquals(0, result.getErrors().size()); + Assert.assertEquals(1, result.getResults().size()); + Assert.assertEquals(10, result.getResults().get(partition).getMessages().size()); + Assert.assertEquals(10,result.getResults().get(partition).getHighWatermark()); + StringDecoder decoder = new StringDecoder(); + int i = 0; + for (KafkaMessage kafkaMessage : result.getResults().get(partition).getMessages()) { + Assert.assertEquals("Key " + i, MessageUtils.decodeKey(kafkaMessage, decoder)); + Assert.assertEquals("Message " + i, MessageUtils.decodePayload(kafkaMessage, decoder)); + i++; + } + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterTests.java new file mode 100644 index 0000000000..3760444eb8 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterTests.java @@ -0,0 +1,126 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; + +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.gs.collections.api.multimap.list.MutableListMultimap; +import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; +import kafka.message.NoCompressionCodec$; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.beans.factory.BeanFactory; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.rule.KafkaRule; +import org.springframework.integration.kafka.serializer.common.StringDecoder; +import org.springframework.integration.kafka.support.KafkaHeaders; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; + +/** + * @author Marius Bogoevici + */ +public class KafkaMessageDrivenChannelAdapterTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaRule getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + @SuppressWarnings("serial") + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + for (int i = 0; i < 5; i++) { + readPartitions.add(new Partition(TEST_TOPIC, i)); + } + + final KafkaMessageListenerContainer kafkaMessageListenerContainer = + new KafkaMessageListenerContainer(connectionFactory, + readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + + int expectedMessageCount = 100; + + final MutableListMultimap receivedData = + new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + + KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = + new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); + + StringDecoder decoder = new StringDecoder(); + kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); + kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { + @Override + public boolean send(Message message) { + latch.countDown(); + return receivedData.put( + (Integer)message.getHeaders().get(KafkaHeaders.PARTITION_ID), + new KeyedMessageWithOffset( + (String)message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), + (String)message.getPayload(), + (Long)message.getHeaders().get(KafkaHeaders.OFFSET), + Thread.currentThread().getName(), + (Integer)message.getHeaders().get(KafkaHeaders.PARTITION_ID))); + } + + + @Override + public boolean send(Message message, long timeout) { + return send(message); + } + }); + + kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); + kafkaMessageDrivenChannelAdapter.start(); + + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessages(100, TEST_TOPIC)); + + latch.await((expectedMessageCount/5000) + 1, TimeUnit.MINUTES); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, 2, 5, 100, expectedMessageCount, readPartitions, 1); + + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests.java new file mode 100644 index 0000000000..e8665f9073 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests.java @@ -0,0 +1,172 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.gs.collections.api.RichIterable; +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.multimap.list.MutableListMultimap; +import com.gs.collections.api.tuple.Pair; +import com.gs.collections.impl.list.mutable.FastList; +import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; +import com.gs.collections.impl.utility.Iterate; + +import kafka.message.NoCompressionCodec$; + +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.beans.factory.BeanFactory; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.rule.KafkaRule; +import org.springframework.integration.kafka.serializer.common.StringDecoder; +import org.springframework.integration.kafka.support.KafkaHeaders; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; + +/** + * @author Marius Bogoevici + */ +public class KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaRule getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + @SuppressWarnings("serial") + public void testLowVolumeLowConcurrency() throws Exception { + + // we will send 300 messages: first 200, then another 100 + // we will start reading from all partitions at offset 100 + int expectedMessageCount = 200; + + createTopic(TEST_TOPIC, 5, 1, 1); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + Map startingOffsets = new HashMap(); + for (int i = 0; i < 5; i++) { + Partition partition = new Partition(TEST_TOPIC, i); + readPartitions.add(partition); + startingOffsets.put(partition, 20L); + } + + final KafkaMessageListenerContainer kafkaMessageListenerContainer = + new KafkaMessageListenerContainer(connectionFactory, + readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + MetadataStoreOffsetManager offsetManager = new MetadataStoreOffsetManager(connectionFactory, startingOffsets); + kafkaMessageListenerContainer.setOffsetManager(offsetManager); + + // we send 100 messages + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessagesInRange(0, 199, TEST_TOPIC)); + + final MutableListMultimap receivedData = + new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + + KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = + new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); + + StringDecoder decoder = new StringDecoder(); + kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); + kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { + @Override + public boolean send(Message message) { + latch.countDown(); + return receivedData.put( + (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID), + new KeyedMessageWithOffset( + (String) message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), + (String) message.getPayload(), + (Long) message.getHeaders().get(KafkaHeaders.OFFSET), + Thread.currentThread().getName(), + (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID))); + } + + + @Override + public boolean send(Message message, long timeout) { + return send(message); + } + }); + + kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); + kafkaMessageDrivenChannelAdapter.start(); + + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessagesInRange(200, 299, TEST_TOPIC)); + + latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, 2, 5, 100, expectedMessageCount, readPartitions, 1); + + // For all received messages + Collection allReceivedMessages = + Iterate.flatCollect(receivedData.keyMultiValuePairsView(), + new Function>, + RichIterable>() { + + @Override + public RichIterable valueOf(Pair> object) { + return object.getTwo(); + } + + }); + + // We extract the sequence value, i.e. "Message xx" + Integer minValueInMessage = FastList.newList(allReceivedMessages).collect(new Function() { + @Override + public Integer valueOf(KeyedMessageWithOffset object) { + return Integer.parseInt(object.getPayload().split(" ")[1]); + } + }).min(); + + // The lowest received value is 100. That is correct, because messages are evenly distributed across partitions + // and we start reading only at partition 200 + assertThat(minValueInMessage, equalTo(100)); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithWrongOffsetTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithWrongOffsetTests.java new file mode 100644 index 0000000000..ad85926afd --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithWrongOffsetTests.java @@ -0,0 +1,319 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.gs.collections.api.RichIterable; +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.multimap.list.MutableListMultimap; +import com.gs.collections.api.tuple.Pair; +import com.gs.collections.impl.list.mutable.FastList; +import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; +import com.gs.collections.impl.utility.Iterate; + +import kafka.message.NoCompressionCodec$; + +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.beans.factory.BeanFactory; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.rule.KafkaRule; +import org.springframework.integration.kafka.serializer.common.StringDecoder; +import org.springframework.integration.kafka.support.KafkaHeaders; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.util.StringUtils; + +/** + * @author Marius Bogoevici + */ +public class KafkaMessageDrivenChannelAdapterWithWrongOffsetTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaRule getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + @SuppressWarnings("serial") + public void testLowVolumeLowConcurrencyResetAtEarliest() throws Exception { + + // we will send 300 messages: first 200, then another 100 + // we will start reading from all partitions at offset 100 + int expectedMessageCount = 200; + + createTopic(TEST_TOPIC, 5, 1, 1); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + Map startingOffsets = new HashMap(); + for (int i = 0; i < 5; i++) { + Partition partition = new Partition(TEST_TOPIC, i); + readPartitions.add(partition); + startingOffsets.put(partition, 900L); + } + + final KafkaMessageListenerContainer kafkaMessageListenerContainer = + new KafkaMessageListenerContainer(connectionFactory, + readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + MetadataStoreOffsetManager offsetManager = new MetadataStoreOffsetManager(connectionFactory, startingOffsets); + kafkaMessageListenerContainer.setOffsetManager(offsetManager); + + // we send 100 messages + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessagesInRange(0, 199, TEST_TOPIC)); + + final MutableListMultimap receivedData = + new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + + KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = + new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); + + StringDecoder decoder = new StringDecoder(); + kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); + + kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { + @Override + public boolean send(Message message) { + latch.countDown(); + return receivedData.put( + (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID), + new KeyedMessageWithOffset( + (String) message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), + (String) message.getPayload(), + (Long) message.getHeaders().get(KafkaHeaders.OFFSET), + Thread.currentThread().getName(), + (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID))); + } + + + @Override + public boolean send(Message message, long timeout) { + return send(message); + } + }); + + kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); + kafkaMessageDrivenChannelAdapter.start(); + + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessagesInRange(200, 299, TEST_TOPIC)); + + latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, 2, 5, 100, expectedMessageCount, readPartitions, 1); + + // For all received messages + Collection allReceivedMessages = + Iterate.flatCollect(receivedData.keyMultiValuePairsView(), + new Function>, RichIterable>() { + + @Override + public RichIterable valueOf(Pair> object) { + return object.getTwo(); + } + + }); + + // We extract the sequence value, i.e. "Message xx" + Integer minValueInMessage = FastList.newList(allReceivedMessages) + .collect(new Function() { + @Override + public Integer valueOf(KeyedMessageWithOffset object) { + return Integer.parseInt(object.getPayload().split(" ")[1]); + } + }).min(); + + // The lowest received value is 0. That is correct, because reading started at 0 after the reset + assertThat(minValueInMessage, equalTo(0)); + } + + @Test + @SuppressWarnings("serial") + public void testLowVolumeLowConcurrencyResetAtLatest() throws Exception { + + // we will send 300 messages: first 200, then another 100 + // we will start reading from all partitions at offset 900, but since that is reset + int expectedMessageCount = 100; + + createTopic(TEST_TOPIC, 5, 1, 1); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + Map startingOffsets = new HashMap(); + for (int i = 0; i < 5; i++) { + Partition partition = new Partition(TEST_TOPIC, i); + readPartitions.add(partition); + startingOffsets.put(partition, 900L); + } + + final KafkaMessageListenerContainer kafkaMessageListenerContainer = + new KafkaMessageListenerContainer(connectionFactory, + readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + final MetadataStoreOffsetManager offsetManager = + new MetadataStoreOffsetManager(connectionFactory, startingOffsets); + offsetManager.setReferenceTimestamp(-1); + final CountDownLatch offsetResetLatch = new CountDownLatch(1); + + // Manually generated proxy for catching the reset event. We only want to send data once we are + // sure that the OffsetManager has been reset. Otherwise the test may not finish (due to pollers starting after send + + OffsetManager offsetManagerWrapper = new OffsetManager() { + + @Override + public void flush() throws IOException { + offsetManager.flush(); + } + + @Override + public void close() throws IOException { + offsetManager.close(); + } + + @Override + public void resetOffsets(Collection partitionsToReset) { + System.out.println("resetting " + offsetResetLatch.getCount() + " partitions: " + + StringUtils.collectionToDelimitedString(partitionsToReset, ",")); + offsetManager.resetOffsets(partitionsToReset); + offsetResetLatch.countDown(); + } + + @Override + public long getOffset(Partition partition) { + long offset = offsetManager.getOffset(partition); + System.out.println("Result for " + partition + " offset: " + offset); + return offset; + } + + @Override + public void updateOffset(Partition partition, long offset) { + offsetManager.updateOffset(partition, offset); + } + + }; + + kafkaMessageListenerContainer.setOffsetManager(offsetManagerWrapper); + + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessagesInRange(0, 199, TEST_TOPIC)); + + final MutableListMultimap receivedData = + new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + + KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = + new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); + + StringDecoder decoder = new StringDecoder(); + kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); + kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { + @Override + public boolean send(Message message) { + latch.countDown(); + return receivedData.put( + (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID), + new KeyedMessageWithOffset( + (String) message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), + (String) message.getPayload(), + (Long) message.getHeaders().get(KafkaHeaders.OFFSET), + Thread.currentThread().getName(), + (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID))); + } + + @Override + public boolean send(Message message, long timeout) { + return send(message); + } + }); + + kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); + kafkaMessageDrivenChannelAdapter.start(); + + // we wait for the reset event first + offsetResetLatch.await(1000, TimeUnit.MILLISECONDS); + Thread.sleep(1000); + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessagesInRange(200, 299, TEST_TOPIC)); + + latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, 2, 5, 100, expectedMessageCount, readPartitions, 1); + + // For all received messages + Collection allReceivedMessages = + Iterate.flatCollect(receivedData.keyMultiValuePairsView(), + new Function>, RichIterable>() { + + @Override + public RichIterable valueOf(Pair> object) { + return object.getTwo(); + } + + }); + + // We extract the sequence value, i.e. "Message xx" + Integer minValueInMessage = FastList.newList(allReceivedMessages) + .collect(new Function() { + + @Override + public Integer valueOf(KeyedMessageWithOffset object) { + return Integer.parseInt(object.getPayload().split(" ")[1]); + } + + }) + .min(); + + // The lowest received value is 0. That is correct, because reading started at 0 after the reset + assertThat(minValueInMessage, equalTo(200)); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker20Tests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker20Tests.java new file mode 100644 index 0000000000..2ef274c065 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker20Tests.java @@ -0,0 +1,60 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ +public class MultiBroker20Tests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(20); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 20, 1); + runMessageListenerTest(100, 20, 100, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 20, 1); + runMessageListenerTest(100, 20, 100, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 20, 1); + runMessageListenerTest(100, 20, 100, 100000, 1, 0, TEST_TOPIC); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker2Tests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker2Tests.java new file mode 100644 index 0000000000..5c16d3a3f1 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker2Tests.java @@ -0,0 +1,103 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.rule.KafkaRule; + +/** + * @author Marius Bogoevici + */ +public class MultiBroker2Tests extends AbstractMessageListenerContainerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(2); + + @Override + public KafkaRule getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + runMessageListenerTest(100, 2, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + runMessageListenerTest(100, 2, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + runMessageListenerTest(100, 2, 5, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testLowVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + runMessageListenerTest(100, 5, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + runMessageListenerTest(100, 5, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + runMessageListenerTest(100, 5, 5, 100000, 1, 0, TEST_TOPIC); + } + + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 2, 1); + runMessageListenerTest(100, 20, 100, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 2, 1); + runMessageListenerTest(100, 20, 100, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 2, 1); + runMessageListenerTest(100, 20, 100, 100000, 1, 0, TEST_TOPIC); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedTests.java new file mode 100644 index 0000000000..b3023bf515 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedTests.java @@ -0,0 +1,102 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ +public class MultiBroker5ReplicatedTests extends AbstractMessageListenerContainerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(5); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 3); + runMessageListenerTest(100, 2, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 3); + runMessageListenerTest(100, 2, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 3); + runMessageListenerTest(100, 2, 5, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testLowVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 3); + runMessageListenerTest(100, 5, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 3); + runMessageListenerTest(100, 5, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 5, 5, 100000, 1, 0, TEST_TOPIC); + } + + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 5, 3); + runMessageListenerTest(100, 20, 100, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 5, 3); + runMessageListenerTest(100, 20, 100, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 5, 3); + runMessageListenerTest(100, 20, 100, 100000, 1, 0, TEST_TOPIC); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedWithBounceTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedWithBounceTests.java new file mode 100644 index 0000000000..6451179d71 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedWithBounceTests.java @@ -0,0 +1,114 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; +import static org.springframework.integration.kafka.util.MessageUtils.decodeKey; +import static org.springframework.integration.kafka.util.MessageUtils.decodePayload; + +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.gs.collections.api.multimap.list.MutableListMultimap; +import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; + +import kafka.serializer.StringDecoder; +import kafka.utils.VerifiableProperties; + +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ + +public class MultiBroker5ReplicatedWithBounceTests extends AbstractMessageListenerContainerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(5); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 3); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + for (int i = 0; i < 5; i++) { + readPartitions.add(new Partition(TEST_TOPIC, i)); + } + final KafkaMessageListenerContainer kafkaMessageListenerContainer = + new KafkaMessageListenerContainer(connectionFactory, + readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + + final int expectedMessageCount = 100; + + createStringProducer(0).send(createMessages(100, TEST_TOPIC)); + + + final MutableListMultimap receivedData = + new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + kafkaMessageListenerContainer.setMessageListener(new MessageListener() { + + @Override + public void onMessage(KafkaMessage message) { + StringDecoder decoder = new StringDecoder(new VerifiableProperties()); + receivedData.put(message.getMetadata().getPartition().getId(), + new KeyedMessageWithOffset(decodeKey(message, decoder), decodePayload(message, decoder), + message.getMetadata().getOffset(), Thread.currentThread().getName(), + message.getMetadata().getPartition().getId())); + latch.countDown(); + } + + }); + + kafkaMessageListenerContainer.start(); + + for (int i = 0; i < 5; i++) { + if (i % 2 == 0) { + kafkaEmbeddedBrokerRule.bounce(i); + } + } + + latch.await(50, TimeUnit.SECONDS); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, 2, 5, 100, expectedMessageCount, readPartitions, 1); + + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5Tests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5Tests.java new file mode 100644 index 0000000000..8bfc351fb6 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5Tests.java @@ -0,0 +1,102 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ +public class MultiBroker5Tests extends AbstractMessageListenerContainerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(5); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 2, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 2, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 2, 5, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testLowVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 5, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 5, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 5, 5, 100000, 1, 0, TEST_TOPIC); + } + + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 5, 1); + runMessageListenerTest(100, 20, 100, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 5, 1); + runMessageListenerTest(100, 20, 100, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 5, 1); + runMessageListenerTest(100, 20, 100, 100000, 1, 0, TEST_TOPIC); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerExternalTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerExternalTests.java new file mode 100644 index 0000000000..32520f9961 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerExternalTests.java @@ -0,0 +1,131 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import java.util.UUID; + +import org.junit.After; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaRule; +import org.springframework.integration.kafka.rule.KafkaRunning; + +/** + * @author Marius Bogoevici + */ +@Ignore +public class SingleBrokerExternalTests extends AbstractMessageListenerContainerTests { + + @ClassRule + public static final KafkaRunning kafkaRunningRule = KafkaRunning.isRunning(); + + @Override + public KafkaRule getKafkaRule() { + return kafkaRunningRule; + } + + @Override + @After + public void cleanUp() { + // do nothing, each topic must be created individually + } + + private String generateTopicName() { + return "test" + UUID.randomUUID().toString().replace("-",""); + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 5, 1, 1); + runMessageListenerTest(100, 2, 5, 100, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + public void testMediumVolumeLowConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 5, 1, 1); + runMessageListenerTest(100, 2, 5, 1000, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 5, 1, 1); + runMessageListenerTest(100, 2, 5, 10000, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + public void testLowVolumeMediumConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 5, 1, 1); + runMessageListenerTest(100, 5, 5, 100, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + public void testMediumVolumeMediumConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 5, 1, 1); + runMessageListenerTest(100, 5, 5, 1000, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 5, 1, 1); + runMessageListenerTest(100, 5, 5, 100000, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 1000, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 10000, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 100000, 1, 0, topicName); + deleteTopic(topicName); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerTests.java new file mode 100644 index 0000000000..963d453a70 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerTests.java @@ -0,0 +1,98 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ +public class SingleBrokerTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(100, 2, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + public void testMediumVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(100, 2, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(100, 2, 5, 10000, 1, 0, TEST_TOPIC); + } + + @Test + public void testLowVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(100, 5, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + public void testMediumVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(100, 5, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(100, 5, 5, 100000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 100000, 1, 0, TEST_TOPIC); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithCompressionTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithCompressionTests.java new file mode 100644 index 0000000000..b4ae51c62a --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithCompressionTests.java @@ -0,0 +1,98 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ +public class SingleBrokerWithCompressionTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(10000, 2, 5, 100, 1, 1, TEST_TOPIC); + } + + @Test + public void testMediumVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(10000, 2, 5, 1000, 1, 1, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(10000, 2, 5, 10000, 1, 1, TEST_TOPIC); + } + + @Test + public void testLowVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(10000, 5, 5, 100, 1, 1, TEST_TOPIC); + } + + @Test + public void testMediumVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(10000, 5, 5, 1000, 1, 1, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(10000, 5, 5, 100000, 1, 1, TEST_TOPIC); + } + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(10000, 20, 100, 1000, 1, 1, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(10000, 20, 100, 10000, 1, 1, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(10000, 20, 100, 100000, 1, 1, TEST_TOPIC); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithPartitionSubsetTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithPartitionSubsetTests.java new file mode 100644 index 0000000000..7f0235122d --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithPartitionSubsetTests.java @@ -0,0 +1,103 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ +public class SingleBrokerWithPartitionSubsetTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 4, 1, 1); + runMessageListenerTest(100, 2, 4, 100, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 4, 1, 1); + runMessageListenerTest(100, 2, 4, 1000, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 4, 1, 1); + runMessageListenerTest(100, 2, 4, 10000, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testLowVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 10, 1, 1); + runMessageListenerTest(100, 5, 10, 100, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 10, 1, 1); + runMessageListenerTest(100, 5, 10, 1000, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 10, 1, 1); + runMessageListenerTest(100, 5, 10, 100000, 2, 0, TEST_TOPIC); + } + + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 1000, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 10000, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 100000, 2, 0, TEST_TOPIC); + } + + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/TestPartitioner.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/TestPartitioner.java new file mode 100644 index 0000000000..46ff1f5e71 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/TestPartitioner.java @@ -0,0 +1,50 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import kafka.producer.Partitioner; +import kafka.utils.VerifiableProperties; + +/** + * @author Marius Bogoevici + */ +public class TestPartitioner implements Partitioner { + + public TestPartitioner(VerifiableProperties properties) { + } + + @Override + public int partition(Object key, int numPartitions) { + if (key != null) { + if (key instanceof Number) { + return ((Number) key).intValue() % numPartitions; + } + else { + try { + return Integer.parseInt(key.toString()); + } + catch (NumberFormatException e) { + return 0; + } + } + } + else { + return 0; + } + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/WrongTopicNameTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/WrongTopicNameTests.java new file mode 100644 index 0000000000..b0ca1e59a0 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/WrongTopicNameTests.java @@ -0,0 +1,90 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.beans.factory.BeanFactory; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.TopicNotFoundException; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.rule.KafkaRule; +import org.springframework.integration.kafka.serializer.common.StringDecoder; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; + +/** + * @author Marius Bogoevici + */ +public class WrongTopicNameTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaRule getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test(expected = TopicNotFoundException.class) + @SuppressWarnings("serial") + public void testWrongTopicNameFails() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + + final KafkaMessageListenerContainer kafkaMessageListenerContainer + = new KafkaMessageListenerContainer(connectionFactory, "WRONG-TOPIC"); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + + KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = + new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); + + StringDecoder decoder = new StringDecoder(); + kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); + kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { + @Override + public boolean send(Message message) { + fail(); + return true; + } + + @Override + public boolean send(Message message, long timeout) { + fail(); + return true; + } + }); + + kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); + kafkaMessageDrivenChannelAdapter.start(); + + + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ZookeeperConfigurationTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ZookeeperConfigurationTests.java new file mode 100644 index 0000000000..3e00ec9311 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ZookeeperConfigurationTests.java @@ -0,0 +1,107 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; +import static org.springframework.integration.kafka.util.MessageUtils.decodeKey; +import static org.springframework.integration.kafka.util.MessageUtils.decodePayload; + +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.gs.collections.api.multimap.list.MutableListMultimap; +import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; +import kafka.serializer.StringDecoder; +import kafka.utils.VerifiableProperties; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.core.Configuration; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.ZookeeperConfiguration; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.support.ZookeeperConnect; + +/** + * @author Marius Bogoevici + */ +public class ZookeeperConfigurationTests extends AbstractMessageListenerContainerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(2); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Override + public Configuration getKafkaConfiguration() { + ZookeeperConnect zookeeperConnect = new ZookeeperConnect(); + zookeeperConnect.setZkConnect(kafkaEmbeddedBrokerRule.getZookeeperConnectionString()); + return new ZookeeperConfiguration(zookeeperConnect); + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + for (int i = 0; i < 5; i++) { + readPartitions.add(new Partition(TEST_TOPIC, i)); + } + final KafkaMessageListenerContainer kafkaMessageListenerContainer = new KafkaMessageListenerContainer(connectionFactory, readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + + int expectedMessageCount = 100; + + final MutableListMultimap receivedData = new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + kafkaMessageListenerContainer.setMessageListener(new MessageListener() { + @Override + public void onMessage(KafkaMessage message) { + StringDecoder decoder = new StringDecoder(new VerifiableProperties()); + receivedData.put(message.getMetadata().getPartition().getId(),new KeyedMessageWithOffset(decodeKey(message, decoder), decodePayload(message, decoder), message.getMetadata().getOffset(), Thread.currentThread().getName(), message.getMetadata().getPartition().getId())); + latch.countDown(); + } + }); + + kafkaMessageListenerContainer.start(); + + createStringProducer(0).send(createMessages(100, TEST_TOPIC)); + + latch.await((expectedMessageCount/5000) + 1, TimeUnit.MINUTES); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, 2, 5, 100, expectedMessageCount, readPartitions, 1); + + } + + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/OutboundTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/OutboundTests.java index 1471b3bac1..fe4ddb2948 100644 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/OutboundTests.java +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/OutboundTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2014 the original author or authors. + * Copyright 2015 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,18 +16,23 @@ package org.springframework.integration.kafka.outbound; -import static org.junit.Assert.assertNotNull; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThat; +import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import kafka.admin.AdminUtils; -import kafka.consumer.ConsumerConfig; +import kafka.api.OffsetRequest; +import kafka.common.TopicExistsException; import kafka.serializer.Decoder; import kafka.serializer.Encoder; - import org.junit.AfterClass; import org.junit.ClassRule; import org.junit.Test; @@ -35,26 +40,27 @@ import org.junit.Test; import org.springframework.context.expression.MapAccessor; import org.springframework.expression.spel.standard.SpelExpressionParser; import org.springframework.expression.spel.support.StandardEvaluationContext; +import org.springframework.integration.kafka.core.DefaultConnectionFactory; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.ZookeeperConfiguration; +import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer; +import org.springframework.integration.kafka.listener.MessageListener; +import org.springframework.integration.kafka.listener.MetadataStoreOffsetManager; import org.springframework.integration.kafka.rule.KafkaRunning; import org.springframework.integration.kafka.serializer.common.StringDecoder; import org.springframework.integration.kafka.serializer.common.StringEncoder; -import org.springframework.integration.kafka.support.ConsumerConfigFactoryBean; -import org.springframework.integration.kafka.support.ConsumerConfiguration; -import org.springframework.integration.kafka.support.ConsumerConnectionProvider; -import org.springframework.integration.kafka.support.ConsumerMetadata; -import org.springframework.integration.kafka.support.KafkaConsumerContext; import org.springframework.integration.kafka.support.KafkaHeaders; import org.springframework.integration.kafka.support.KafkaProducerContext; -import org.springframework.integration.kafka.support.MessageLeftOverTracker; import org.springframework.integration.kafka.support.ProducerConfiguration; import org.springframework.integration.kafka.support.ProducerFactoryBean; import org.springframework.integration.kafka.support.ProducerMetadata; import org.springframework.integration.kafka.support.ZookeeperConnect; -import org.springframework.messaging.Message; +import org.springframework.integration.kafka.util.MessageUtils; import org.springframework.messaging.support.MessageBuilder; /** * @author Gary Russell + * @author Marius Bogoevici * @since 1.0 * */ @@ -76,9 +82,42 @@ public class OutboundTests { @Test public void testAsyncProducerFlushed() throws Exception { - KafkaConsumerContext consumerContext = createConsumer(); - // pre-consume to start the receiver because the high-level API doesn't support --from-beginning - consumerContext.receive(); + + // create the topic + + try { + AdminUtils.createTopic(kafkaRunning.getZkClient(), TOPIC, 1, 1, new Properties()); + } + catch (TopicExistsException e) { + // do nothing + } + + final String suffix = UUID.randomUUID().toString(); + + ZookeeperConfiguration configuration = new ZookeeperConfiguration(new ZookeeperConnect()); + DefaultConnectionFactory connectionFactory = new DefaultConnectionFactory(configuration); + connectionFactory.afterPropertiesSet(); + final KafkaMessageListenerContainer kafkaMessageListenerContainer = new KafkaMessageListenerContainer(connectionFactory, TOPIC); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(1); + MetadataStoreOffsetManager offsetManager = new MetadataStoreOffsetManager(connectionFactory); + // start reading at the end of the + offsetManager.setReferenceTimestamp(OffsetRequest.LatestTime()); + kafkaMessageListenerContainer.setOffsetManager(offsetManager); + final Decoder decoder = new StringDecoder(); + + int expectedMessageCount = 2; + final List payloads = new ArrayList(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + kafkaMessageListenerContainer.setMessageListener(new MessageListener() { + @Override + public void onMessage(KafkaMessage message) { + payloads.add(MessageUtils.decodePayload(message, decoder)); + latch.countDown(); + } + }); + + kafkaMessageListenerContainer.start(); KafkaProducerContext kafkaProducerContext = new KafkaProducerContext(); ProducerMetadata producerMetadata = new ProducerMetadata(TOPIC); @@ -97,7 +136,7 @@ public class OutboundTests { kafkaProducerContext.setProducerConfigurations(Collections.singletonMap(TOPIC, config)); KafkaProducerMessageHandler handler = new KafkaProducerMessageHandler(kafkaProducerContext); - handler.handleMessage(MessageBuilder.withPayload("foo") + handler.handleMessage(MessageBuilder.withPayload("foo"+suffix) .setHeader(KafkaHeaders.MESSAGE_KEY, "3") .setHeader(KafkaHeaders.TOPIC, TOPIC) .build()); @@ -108,45 +147,19 @@ public class OutboundTests { StandardEvaluationContext evaluationContext = new StandardEvaluationContext(); evaluationContext.addPropertyAccessor(new MapAccessor()); handler.setIntegrationEvaluationContext(evaluationContext); - handler.handleMessage(MessageBuilder.withPayload("bar") + handler.handleMessage(MessageBuilder.withPayload("bar"+suffix) .setHeader("foo", "3") .setHeader("bar", TOPIC) .build()); kafkaProducerContext.stop(); - Message>>> received = consumerContext.receive(); - assertNotNull(received); - if (((Map) received.getPayload()).size() < 2) { - received = consumerContext.receive(); - assertNotNull(received); + latch.await(1000, TimeUnit.MILLISECONDS); + assertThat(latch.getCount(), equalTo(0L)); + for (String payload : payloads) { + assertThat(payload, endsWith(suffix)); } - consumerContext.destroy(); - } - - private KafkaConsumerContext createConsumer() throws Exception { - KafkaConsumerContext consumerContext = new KafkaConsumerContext(); - ZookeeperConnect zookeeperConnect = new ZookeeperConnect(); - zookeeperConnect.setZkConnect("localhost:2181"); - consumerContext.setZookeeperConnect(zookeeperConnect); - ConsumerMetadata consumerMetadata = new ConsumerMetadata(); - consumerMetadata.setGroupId("foo"); - Decoder decoder = new StringDecoder(); - consumerMetadata.setValueDecoder(decoder); - consumerMetadata.setKeyDecoder(decoder); - consumerMetadata.setTopicStreamMap(Collections.singletonMap(TOPIC, 1)); - Properties consumerProps = new Properties(); - consumerProps.put("consumer.timeout.ms", "500"); - ConsumerConfigFactoryBean consumerConfigFactoryBean = new ConsumerConfigFactoryBean( - consumerMetadata, zookeeperConnect, consumerProps); - ConsumerConfig consumerConfig = consumerConfigFactoryBean.getObject(); - ConsumerConnectionProvider consumerConnectionProvider = new ConsumerConnectionProvider(consumerConfig); - MessageLeftOverTracker messageLeftOverTracker = new MessageLeftOverTracker(); - ConsumerConfiguration cConfig = new ConsumerConfiguration(consumerMetadata, - consumerConnectionProvider, messageLeftOverTracker); - cConfig.setMaxMessages(1); - consumerContext.setConsumerConfigurations(Collections.singletonMap("foo", cConfig)); - return consumerContext; + kafkaMessageListenerContainer.stop(); } } diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaEmbedded.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaEmbedded.java new file mode 100644 index 0000000000..1ade044eef --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaEmbedded.java @@ -0,0 +1,177 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.rule; + +import static scala.collection.JavaConversions.asScalaBuffer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.impl.list.mutable.FastList; +import com.gs.collections.impl.utility.ListIterate; +import kafka.Kafka; +import kafka.server.KafkaConfig; +import kafka.server.KafkaServer; +import kafka.utils.SystemTime$; +import kafka.utils.TestUtils; +import kafka.utils.TestZKUtils; +import kafka.utils.Utils; +import kafka.utils.ZKStringSerializer$; +import kafka.zk.EmbeddedZookeeper; +import org.I0Itec.zkclient.ZkClient; +import org.I0Itec.zkclient.exception.ZkInterruptedException; +import org.junit.rules.ExternalResource; +import scala.collection.JavaConversions; + +import org.springframework.integration.kafka.core.BrokerAddress; + +/** + * @author Marius Bogoevici + */ +@SuppressWarnings("serial") +public class KafkaEmbedded extends ExternalResource implements KafkaRule { + + private int count; + + private boolean controlledShutdown; + + private List kafkaPorts; + + private List kafkaServers; + + private EmbeddedZookeeper zookeeper; + + private ZkClient zookeeperClient; + + @SuppressWarnings("unchecked") + public KafkaEmbedded(int count, boolean controlledShutdown) { + this.count = count; + this.controlledShutdown = controlledShutdown; + this.kafkaPorts = JavaConversions.asJavaList((scala.collection.immutable.List) TestUtils.choosePorts(count)); + } + + public KafkaEmbedded(int count) { + this(count, false); + } + + @Override + protected void before() throws Throwable { + zookeeper = new EmbeddedZookeeper(TestZKUtils.zookeeperConnect()); + int zkConnectionTimeout = 6000; + int zkSessionTimeout = 6000; + zookeeperClient = new ZkClient(TestZKUtils.zookeeperConnect(), zkSessionTimeout, zkConnectionTimeout, ZKStringSerializer$.MODULE$); + kafkaServers = new ArrayList(); + for (int i = 0; i < count; i++) { + Properties brokerConfigProperties = TestUtils.createBrokerConfig(i, kafkaPorts.get(i)); + brokerConfigProperties.put("controlled.shutdown.enable", Boolean.toString(controlledShutdown)); + KafkaServer server = TestUtils.createServer(new KafkaConfig(brokerConfigProperties), SystemTime$.MODULE$); + kafkaServers.add(server); + } + } + + @Override + protected void after() { + for (KafkaServer kafkaServer : kafkaServers) { + try { + kafkaServer.shutdown(); + } + catch (Exception e) { + // do nothing + } + try { + Utils.rm(kafkaServer.config().logDirs()); + } + catch (Exception e) { + // do nothing + } + } + try { + zookeeperClient.close(); + } + catch (ZkInterruptedException e) { + // do nothing + } + try { + zookeeper.shutdown(); + } + catch (Exception e) { + // do nothing + } + } + + public List getKafkaServers() { + return kafkaServers; + } + + public KafkaServer getKafkaServer(int id) { + return kafkaServers.get(id); + } + + public EmbeddedZookeeper getZookeeper() { + return zookeeper; + } + + @Override + public ZkClient getZkClient() { + return zookeeperClient; + } + + public String getZookeeperConnectionString() { + return zookeeper.connectString(); + } + + @Override + public List getBrokerAddresses() { + return ListIterate.collect(kafkaServers, new Function() { + @Override + public BrokerAddress valueOf(KafkaServer kafkaServer) { + return new BrokerAddress(kafkaServer.config().hostName(), kafkaServer.config().port()); + } + }); + } + + public void bounce(BrokerAddress brokerAddress) { + for (KafkaServer kafkaServer : getKafkaServers()) { + if (brokerAddress.equals(new BrokerAddress(kafkaServer.config().hostName(), kafkaServer.config().port()))) { + kafkaServer.shutdown(); + } + } + } + + public void bounce(int index) { + kafkaServers.get(index).shutdown(); + TestUtils.waitUntilMetadataIsPropagated(asScalaBuffer(kafkaServers), "test-topic", 0, 5000L); + } + + @Override + public String getBrokersAsString() { + return FastList.newList(getBrokerAddresses()).collect(new Function() { + @Override + public String valueOf(BrokerAddress object) { + return object.getHost() + ":" + object.getPort(); + } + }).makeString(","); + } + + @Override + public boolean isEmbedded() { + return true; + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRule.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRule.java new file mode 100644 index 0000000000..9b6bac37d2 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRule.java @@ -0,0 +1,44 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.rule; + +import java.util.List; + +import kafka.server.KafkaServer; +import org.I0Itec.zkclient.ZkClient; +import org.junit.rules.TestRule; + +import org.springframework.integration.kafka.core.BrokerAddress; + +/** + * Common functionality for the Kafka JUnit rules + * + * @author Marius Bogoevici + */ +public interface KafkaRule extends TestRule { + + ZkClient getZkClient(); + + List getBrokerAddresses(); + + String getBrokersAsString(); + + boolean isEmbedded(); + + List getKafkaServers(); +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRunning.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRunning.java index fdb4495b2b..e71dd0b663 100644 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRunning.java +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRunning.java @@ -15,6 +15,16 @@ */ package org.springframework.integration.kafka.rule; +import java.util.List; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.impl.block.factory.Functions; +import com.gs.collections.impl.list.mutable.FastList; +import com.gs.collections.impl.utility.ListIterate; +import kafka.cluster.Broker; +import kafka.server.KafkaServer; +import kafka.utils.ZKStringSerializer$; +import kafka.utils.ZkUtils; import org.I0Itec.zkclient.ZkClient; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -22,33 +32,33 @@ import org.junit.Assume; import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.junit.runners.model.Statement; +import scala.collection.JavaConversions; +import scala.collection.Seq; +import org.springframework.integration.kafka.core.BrokerAddress; import org.springframework.integration.kafka.core.ZookeeperConnectDefaults; -import kafka.utils.ZKStringSerializer$; -import kafka.utils.ZkUtils; - /** - *

- * A rule that prevents integration tests from failing if the Kafka server is not running or not + * * A rule that prevents integration tests from failing if the Kafka server is not running or not * accessible. If the Kafka server is not running in the background all the tests here will simply be skipped because * of a violated assumption (showing as successful). - *

* The rule can be declared as static so that it only has to check once for all tests in the enclosing test case, but * there isn't a lot of overhead in making it non-static. * * @author Dave Syer * @author Artem Bilan * @author Gary Russell - * + * @author Marius Bogoevici * @since 1.0 */ -public class KafkaRunning extends TestWatcher { +public class KafkaRunning extends TestWatcher implements KafkaRule { private static final String ZOOKEEPER_CONNECT_STRING = ZookeeperConnectDefaults.ZK_CONNECT; private static final Log logger = LogFactory.getLog(KafkaRunning.class); + private ZkClient zkClient; + /** * @return a new rule that assumes an existing running broker */ @@ -56,17 +66,42 @@ public class KafkaRunning extends TestWatcher { return new KafkaRunning(); } - private ZkClient zkClient; - public ZkClient getZkClient() { return zkClient; } + @Override + @SuppressWarnings("serial") + public List getBrokerAddresses() { + Seq allBrokersInCluster = ZkUtils.getAllBrokersInCluster(zkClient); + return ListIterate.collect(JavaConversions.asJavaList(allBrokersInCluster), new Function() { + @Override + public BrokerAddress valueOf(Broker broker) { + return new BrokerAddress(broker.host(), broker.port()); + } + }); + } + + @Override + public String getBrokersAsString() { + return FastList.newList(getBrokerAddresses()).collect(Functions.getToString()).makeString(","); + } + + @Override + public boolean isEmbedded() { + return false; + } + + @Override + public List getKafkaServers() { + throw new UnsupportedOperationException("Not supported on the external rule"); + } + @Override public Statement apply(Statement base, Description description) { try { this.zkClient = new ZkClient(ZOOKEEPER_CONNECT_STRING, 1000, 1000, ZKStringSerializer$.MODULE$); - if (ZkUtils.getAllBrokersInCluster(zkClient).size() == 0) { + if (getBrokerAddresses().size() == 0) { throw new IllegalStateException("No running Kafka brokers"); } }