diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/AbstractConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/AbstractConfiguration.java new file mode 100644 index 0000000000..b1ed97f897 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/AbstractConfiguration.java @@ -0,0 +1,72 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import java.util.List; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +/** + * Default implementation of a {@link Configuration}, storing the default topic and partitions explicitly. + * + * Implementors must provide a strategy for retrieving the seed brokers. + * + * @author Marius Bogoevici + */ +public abstract class AbstractConfiguration implements InitializingBean, Configuration { + + private List defaultPartitions; + + private String defaultTopic; + + @Override + public void afterPropertiesSet() throws Exception { + Assert.isTrue(CollectionUtils.isEmpty(defaultPartitions) || StringUtils.isEmpty(defaultTopic) + , "A list of default partitions or a default topic may be specified, but not both"); + } + + @Override + public final List getBrokerAddresses() { + return doGetBrokerAddresses(); + } + + protected abstract List doGetBrokerAddresses(); + + @Override + public List getDefaultPartitions() { + return defaultPartitions; + } + + public void setDefaultPartitions(List defaultPartitions) { + this.defaultPartitions = defaultPartitions; + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + public void setDefaultTopic(String defaultTopic) { + this.defaultTopic = defaultTopic; + } + + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddress.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddress.java new file mode 100644 index 0000000000..9bdcc893f7 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddress.java @@ -0,0 +1,102 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import org.springframework.util.StringUtils; + +/** + * Encapsulates the address of a Kafka broker. + * + * @author Marius Bogoevici + */ +public class BrokerAddress { + + public static final int DEFAULT_PORT = 9092; + + private final String host; + + private final int port; + + public BrokerAddress(String host, int port) { + if (StringUtils.isEmpty(host)) { + throw new IllegalArgumentException("Host cannot be empty"); + } + this.host = host; + this.port = port; + } + + public BrokerAddress(String host) { + this(host, DEFAULT_PORT); + } + + public static BrokerAddress fromAddress(String address) { + String[] split = address.split(":"); + if (split.length == 0 || split.length > 2) { + throw new IllegalArgumentException("Expected format [:]"); + } + if (split.length == 2) { + return new BrokerAddress(split[0], Integer.parseInt(split[1])); + } + else { + return new BrokerAddress(split[0]); + } + + } + + public String getHost() { + return host; + } + + public int getPort() { + return port; + } + + @Override + public int hashCode() { + return 31 * host.hashCode() + port; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + BrokerAddress brokerAddress = (BrokerAddress) o; + + if (port != brokerAddress.port) { + return false; + } + if (!host.equals(brokerAddress.host)) { + return false; + } + + return true; + } + + @Override + public String toString() { + return host + ":" + port; + } + +} + + diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddressListConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddressListConfiguration.java new file mode 100644 index 0000000000..e64e007d63 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddressListConfiguration.java @@ -0,0 +1,40 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import java.util.List; + +/** + * Kafka {@link Configuration} where the seed brokers are set up explicitly. + * + * @author Marius Bogoevici + */ +public class BrokerAddressListConfiguration extends AbstractConfiguration { + + private final List brokerAddresses; + + public BrokerAddressListConfiguration(List brokerAddresses) { + this.brokerAddresses = brokerAddresses; + } + + @Override + protected List doGetBrokerAddresses() { + return brokerAddresses; + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Configuration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Configuration.java new file mode 100644 index 0000000000..73c58b7ec0 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Configuration.java @@ -0,0 +1,47 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import java.util.List; + +/** + * Used to configure a {@link DefaultConnectionFactory}. Provides a list of seed brokers. + * + * @author Marius Bogoevici + */ +public interface Configuration { + + /** + * The list of seed broker addresses used by this Configuration. + * @return the broker addresses + */ + List getBrokerAddresses(); + + /** + * A list of default partitions to perform operations on. + * @return the list of partitions + */ + List getDefaultPartitions(); + + /** + * A default topic to perform operations on. + * @return a topic name + */ + String getDefaultTopic(); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Connection.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Connection.java new file mode 100644 index 0000000000..7e69a13c5e --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Connection.java @@ -0,0 +1,91 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import java.util.Map; + +import kafka.api.OffsetRequest; + +/** + * A connection to a Kafka broker. + * + * @author Marius Bogoevici + */ +public interface Connection { + + /** + * Fetch data from a Kafka broker. + * @param fetchRequests a list of fetch operations + * @return message batches, indexed by partition + * @throws ConsumerException the ConsumerException if any underlying error + */ + Result fetch(FetchRequest... fetchRequests) throws ConsumerException; + + /** + * Fetch an actual offset in the partition, immediately before the given reference time, + * or the smallest and largest value, respectively, if the special values -1 + * ({@link OffsetRequest#LatestTime()}) and -2 ({@link OffsetRequest#LatestTime()}) + * are used . To be used to position the initial offset of a read operation. + * @param referenceTime The returned values will be before this time, if they exist. The special + * values -2 ({@link OffsetRequest#LatestTime()}) + * and -1 ({@link OffsetRequest#LatestTime()}) are supported. + * @param partitions the offsets, indexed by {@link Partition} + * @return any errors, an empty {@link Result} in case of success + * @throws ConsumerException the ConsumerException if any underlying error + */ + Result fetchInitialOffset(long referenceTime, Partition... partitions) throws ConsumerException; + + /** + * Fetch offsets from the native Kafka offset management system. + * @param consumerId the id of the consumer + * @param partitions the list of partitions whose offsets are queried for + * @return any errors, an empty {@link Result} in case of success + * @throws ConsumerException the ConsumerException if any underlying error + */ + Result fetchStoredOffsetsForConsumer(String consumerId, Partition... partitions) throws ConsumerException; + + /** + * Update offsets in the native Kafka offset management system. + * @param consumerId the id of the consumer + * @param offsets the offsets, indexed by {@link Partition} + * @return any errors, an empty {@link Result} in case of success + * @throws ConsumerException the ConsumerException if any underlying error + */ + Result commitOffsetsForConsumer(String consumerId, Map offsets) throws ConsumerException; + + /** + * Retrieve the leader broker addresses for all the partitions in the given topics. + * @param topics the topics whose partitions we query for + * @return broker addresses, indexed by {@link Partition} + * @throws ConsumerException the ConsumerException if any underlying error + */ + Result findLeaders(String... topics) throws ConsumerException; + + + /** + * The broker address for this consumer + * @return broker address + */ + BrokerAddress getBrokerAddress(); + + /** + * Closes the connection to the broker. No further operations are permitted. + */ + void close(); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConnectionFactory.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConnectionFactory.java new file mode 100644 index 0000000000..3791ca8061 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConnectionFactory.java @@ -0,0 +1,67 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import java.util.Collection; +import java.util.Map; + +/** + * Creates Kafka connections and retrieves metadata for topics and partitions. + * + * @author Marius Bogoevici + * + */ +public interface ConnectionFactory { + + /** + * Create a connection to a Kafka broker, caching it internally + * @param brokerAddress a broker address + * @return a working connection + */ + Connection connect(BrokerAddress brokerAddress); + + /** + * Retrieve the leaders for a set of partitions. + * @param partitions whose leaders are queried + * @return the broker associated with the provided topic and partition + */ + Map getLeaders(Iterable partitions); + + /** + * Return the leader for a single partition + * @param partition the partition whose leader is queried + * @return the leader's address + */ + BrokerAddress getLeader(Partition partition); + + /** + * Refresh the broker connections and partition leader map. To be called when the topology changes + * are detected (i.e. brokers leave and/or partition leaders change) and that results in fetch errors, + * for instance. + * @param topics the topics for which to refresh the leaders + */ + void refreshLeaders(Collection topics); + + /** + * Retrieves the partitions of a given topic + * @param topic the topic to query for + * @return a list of partitions + */ + Collection getPartitions(String topic); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConsumerException.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConsumerException.java new file mode 100644 index 0000000000..49d9698990 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConsumerException.java @@ -0,0 +1,41 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import kafka.javaapi.consumer.SimpleConsumer; + +/** + * Wraps exceptions thrown by {@link SimpleConsumer} calls. + * + * Because the client is originally written in Scala, checked exceptions are missing from the method signatures + * and cannot be caught directly. + * + * @author Marius Bogoevici + */ +@SuppressWarnings("serial") +public class ConsumerException extends RuntimeException { + + public ConsumerException(Throwable cause) { + super(cause); + } + + public ConsumerException(String message) { + super(message); + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnection.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnection.java new file mode 100644 index 0000000000..4c6b0191aa --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnection.java @@ -0,0 +1,322 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.block.function.Function2; +import com.gs.collections.api.tuple.Pair; +import com.gs.collections.impl.list.mutable.FastList; +import com.gs.collections.impl.tuple.Tuples; +import com.gs.collections.impl.utility.LazyIterate; +import com.gs.collections.impl.utility.MapIterate; +import kafka.api.FetchRequestBuilder; +import kafka.api.PartitionOffsetRequestInfo; +import kafka.cluster.Broker; +import kafka.common.ErrorMapping; +import kafka.common.OffsetMetadataAndError; +import kafka.common.TopicAndPartition; +import kafka.javaapi.FetchResponse; +import kafka.javaapi.OffsetCommitRequest; +import kafka.javaapi.OffsetCommitResponse; +import kafka.javaapi.OffsetFetchRequest; +import kafka.javaapi.OffsetFetchResponse; +import kafka.javaapi.OffsetRequest; +import kafka.javaapi.OffsetResponse; +import kafka.javaapi.PartitionMetadata; +import kafka.javaapi.TopicMetadata; +import kafka.javaapi.TopicMetadataRequest; +import kafka.javaapi.TopicMetadataResponse; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.javaapi.message.ByteBufferMessageSet; +import kafka.message.MessageAndOffset; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.util.Assert; + +/** + * A connection to a Kafka broker. + * + * @author Marius Bogoevici + */ +public class DefaultConnection implements Connection { + + private static Log log = LogFactory.getLog(DefaultConnection.class); + + private final AtomicInteger correlationIdCounter = new AtomicInteger(new Random(new Date().getTime()).nextInt()); + + private final SimpleConsumer simpleConsumer; + + private final BrokerAddress brokerAddress; + + private int minBytes; + + private int maxWait; + + public DefaultConnection(BrokerAddress brokerAddress, String clientId, int bufferSize, int soTimeout, + int minBytes, int maxWait) { + this.brokerAddress = brokerAddress; + this.minBytes = minBytes; + this.maxWait = maxWait; + this.simpleConsumer = + new SimpleConsumer(brokerAddress.getHost(), brokerAddress.getPort(), soTimeout, bufferSize, clientId); + } + + /** + * @see Connection#getBrokerAddress() + */ + @Override + public BrokerAddress getBrokerAddress() { + return brokerAddress; + } + + @Override + public void close() { + this.simpleConsumer.close(); + } + + /** + * Fetche data from Kafka. + * @return a combination of messages and errors, depending on whether the invocation was successful or not + * @throws ConsumerException the ConsumerException if any underlying error + */ + @Override + public Result fetch(FetchRequest... requests) throws ConsumerException { + FetchRequestBuilder fetchRequestBuilder = new FetchRequestBuilder(); + for (FetchRequest request : requests) { + Partition partition = request.getPartition(); + long offset = request.getOffset(); + int maxSize = request.getMaxSizeInBytes(); + fetchRequestBuilder.addFetch(partition.getTopic(), partition.getId(), offset, maxSize); + } + FetchResponse fetchResponse; + try { + fetchResponse = this.simpleConsumer.fetch(fetchRequestBuilder.maxWait(maxWait).minBytes(minBytes).build()); + } + catch (Exception e) { + throw new ConsumerException(e); + } + ResultBuilder resultBuilder = new ResultBuilder(); + for (final FetchRequest request : requests) { + Partition partition = request.getPartition(); + if (log.isDebugEnabled()) { + log.debug("Reading from " + partition + "@" + request.getOffset()); + } + short errorCode = fetchResponse.errorCode(partition.getTopic(), partition.getId()); + if (ErrorMapping.NoError() == errorCode) { + ByteBufferMessageSet messageSet = fetchResponse.messageSet(partition.getTopic(), partition.getId()); + List kafkaMessages = LazyIterate.collect(messageSet, + new ConvertToKafkaMessageFunction(request)).toList(); + long highWatermark = fetchResponse.highWatermark(partition.getTopic(), partition.getId()); + resultBuilder.add(partition).withResult(new KafkaMessageBatch(partition, kafkaMessages, highWatermark)); + } + else { + resultBuilder.add(partition).withError(errorCode); + } + } + return resultBuilder.build(); + } + + @Override + public Result fetchStoredOffsetsForConsumer(String consumerId, Partition... partitions) throws + ConsumerException { + FastList topicsAndPartitions = FastList.newList(Arrays.asList(partitions)) + .collect(new ConvertToTopicAndPartitionFunction()); + OffsetFetchRequest offsetFetchRequest = new OffsetFetchRequest(consumerId, topicsAndPartitions, + kafka.api.OffsetFetchRequest.CurrentVersion(), createCorrelationId(), simpleConsumer.clientId()); + OffsetFetchResponse offsetFetchResponse = null; + try { + offsetFetchResponse = simpleConsumer.fetchOffsets(offsetFetchRequest); + } + catch (Exception e) { + throw new ConsumerException(e); + } + ResultBuilder resultBuilder = new ResultBuilder(); + for (Partition partition : partitions) { + OffsetMetadataAndError offsetMetadataAndError + = offsetFetchResponse.offsets().get(new TopicAndPartition(partition.getTopic(), partition.getId())); + short errorCode = offsetMetadataAndError.error(); + if (ErrorMapping.NoError() == errorCode) { + resultBuilder.add(partition).withResult(offsetMetadataAndError.offset()); + } + else { + resultBuilder.add(partition).withError(errorCode); + } + } + return resultBuilder.build(); + } + + /** + * @see Connection#fetchInitialOffset(long, Partition...) + */ + @Override + public Result fetchInitialOffset(long referenceTime, Partition... partitions) throws + ConsumerException { + Assert.isTrue(partitions.length > 0, "Must provide at least one partition"); + Map infoMap = new HashMap(); + for (Partition partition : partitions) { + infoMap.put(new TopicAndPartition(partition.getTopic(), partition.getId()), + new PartitionOffsetRequestInfo(referenceTime, 1)); + } + OffsetRequest offsetRequest = + new OffsetRequest(infoMap, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId()); + OffsetResponse offsetResponse = null; + try { + offsetResponse = simpleConsumer.getOffsetsBefore(offsetRequest); + } + catch (Exception e) { + throw new ConsumerException(e); + } + ResultBuilder resultBuilder = new ResultBuilder(); + for (Partition partition : partitions) { + short errorCode = offsetResponse.errorCode(partition.getTopic(), partition.getId()); + if (ErrorMapping.NoError() == errorCode) { + long[] offsets = offsetResponse.offsets(partition.getTopic(), partition.getId()); + if (offsets.length == 0) { + // normally, we shouldn't get here - either an offset has been returned, + // or an error. However, in case something went wrong, the check protects against an + // ArrayIndexOutOfBoundsException + throw new ConsumerException("Inconsistent response: no error has been returned, " + + "but no offsets either"); + } + resultBuilder.add(partition).withResult(offsets[0]); + } + else { + resultBuilder.add(partition).withError(errorCode); + } + } + return resultBuilder.build(); + } + + @Override + public Result commitOffsetsForConsumer(String consumerId, Map offsets) + throws ConsumerException { + Map requestInfo = + MapIterate.collect(offsets, new CreateRequestInfoMapEntryFunction()); + OffsetCommitResponse offsetCommitResponse = null; + try { + offsetCommitResponse = simpleConsumer.commitOffsets( + new OffsetCommitRequest(consumerId, requestInfo, kafka.api.OffsetCommitRequest.CurrentVersion(), + createCorrelationId(), simpleConsumer.clientId())); + } + catch (Exception e) { + throw new ConsumerException(e); + } + ResultBuilder resultBuilder = new ResultBuilder(); + for (TopicAndPartition topicAndPartition : requestInfo.keySet()) { + if (offsetCommitResponse.errors().containsKey(topicAndPartition)) { + Partition partition = new Partition(topicAndPartition.topic(), topicAndPartition.partition()); + resultBuilder.add(partition).withError((Short) offsetCommitResponse.errors().get(topicAndPartition)); + } + } + return resultBuilder.build(); + } + + /** + * @see Connection#findLeaders(String...) + */ + @Override + public Result findLeaders(String... topics) throws ConsumerException { + TopicMetadataRequest topicMetadataRequest = + new TopicMetadataRequest(Arrays.asList(topics), createCorrelationId()); + TopicMetadataResponse topicMetadataResponse = null; + try { + topicMetadataResponse = simpleConsumer.send(topicMetadataRequest); + } + catch (Exception e) { + throw new ConsumerException(e); + } + ResultBuilder resultBuilder = new ResultBuilder(); + for (TopicMetadata topicMetadata : topicMetadataResponse.topicsMetadata()) { + if (topicMetadata.errorCode() != ErrorMapping.NoError()) { + resultBuilder.add(new Partition(topicMetadata.topic(), -1)).withError(topicMetadata.errorCode()); + } + else { + for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) { + Partition partition = new Partition(topicMetadata.topic(), partitionMetadata.partitionId()); + if (ErrorMapping.NoError() == partitionMetadata.errorCode()) { + Broker leader = partitionMetadata.leader(); + BrokerAddress result = new BrokerAddress(leader.host(), leader.port()); + resultBuilder.add(partition).withResult(result); + } + else { + resultBuilder.add(partition).withError(partitionMetadata.errorCode()); + } + } + + } + } + return resultBuilder.build(); + } + + + + /** + * Creat a pseudo-unique correlation id for requests and responses + * @return correlation id + */ + private Integer createCorrelationId() { + return correlationIdCounter.incrementAndGet(); + } + + @SuppressWarnings("serial") + private static class ConvertToKafkaMessageFunction implements Function { + private final FetchRequest request; + + public ConvertToKafkaMessageFunction(FetchRequest request) { + this.request = request; + } + + @Override + public KafkaMessage valueOf(MessageAndOffset messageAndOffset) { + return new KafkaMessage(messageAndOffset.message(), + new KafkaMessageMetadata(request.getPartition(), messageAndOffset.offset(), + messageAndOffset.nextOffset())); + } + } + + @SuppressWarnings("serial") + private static class ConvertToTopicAndPartitionFunction implements Function { + + @Override + public TopicAndPartition valueOf(Partition partition) { + return new TopicAndPartition(partition.getTopic(), partition.getId()); + } + + } + + @SuppressWarnings("serial") + private static class CreateRequestInfoMapEntryFunction + implements Function2> { + + @Override + public Pair value(Partition partition, Long offset) { + return Tuples.pair(new TopicAndPartition(partition.getTopic(), partition.getId()), + new OffsetMetadataAndError(offset, OffsetMetadataAndError.NoMetadata(), ErrorMapping.NoError())); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnectionFactory.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnectionFactory.java new file mode 100644 index 0000000000..922e04519f --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnectionFactory.java @@ -0,0 +1,260 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.impl.block.factory.Functions; +import com.gs.collections.impl.list.mutable.FastList; +import com.gs.collections.impl.map.mutable.UnifiedMap; +import com.gs.collections.impl.utility.ListIterate; +import kafka.client.ClientUtils$; +import kafka.common.KafkaException; +import kafka.javaapi.PartitionMetadata; +import kafka.javaapi.TopicMetadata; +import kafka.javaapi.TopicMetadataResponse; +import scala.collection.JavaConversions; + +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.context.SmartLifecycle; +import org.springframework.util.Assert; + +/** + * Default implementation of {@link ConnectionFactory} + * + * @author Marius Bogoevici + */ +public class DefaultConnectionFactory implements InitializingBean, ConnectionFactory, DisposableBean { + + private final GetBrokersByPartitionFunction getBrokersByPartitionFunction = new GetBrokersByPartitionFunction(); + + private final ConnectionInstantiationFunction connectionInstantiationFunction = new ConnectionInstantiationFunction(); + + private final Configuration configuration; + + private final AtomicReference partitionBrokerMapReference = new AtomicReference(); + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + + private String clientId = KafkaConsumerDefaults.GROUP_ID; + + private int minBytes = KafkaConsumerDefaults.MIN_FETCH_BYTES; + + private int maxWait = KafkaConsumerDefaults.MAX_WAIT_TIME_IN_MS; + + private int bufferSize = KafkaConsumerDefaults.SOCKET_BUFFER_SIZE_INT; + + private int socketTimeout = KafkaConsumerDefaults.SOCKET_TIMEOUT_INT; + + private int fetchMetadataTimeout = KafkaConsumerDefaults.FETCH_METADATA_TIMEOUT; + + private final UnifiedMap kafkaBrokersCache = UnifiedMap.newMap(); + + public DefaultConnectionFactory(Configuration configuration) { + this.configuration = configuration; + } + + public Configuration getConfiguration() { + return configuration; + } + + @Override + public void afterPropertiesSet() throws Exception { + Assert.notNull(this.configuration, "Kafka configuration cannot be empty"); + this.refreshLeaders(configuration.getDefaultTopic() == null ? Collections.emptyList() : + Collections.singletonList(configuration.getDefaultTopic())); + } + + @Override + public void destroy() throws Exception { + for (Connection connection : kafkaBrokersCache) { + connection.close(); + } + } + + /** + * The minimum amount of data that a server fetch operation will wait for before returning, + * unless {@code maxWaitTimeInMs} has elapsed. + * In conjunction with {@link DefaultConnectionFactory#setMaxWait(int)}, controls latency + * and throughput. + * Smaller values increase responsiveness, but may increase the number of poll operations, + * potentially reducing throughput and increasing CPU consumption. + * @param minBytes the amount of data to fetch + */ + public void setMinBytes(int minBytes) { + this.minBytes = minBytes; + } + + /** + * The maximum amount of time that a server fetch operation will wait before returning + * (unless {@code minFetchSizeInBytes}) are available. + * In conjunction with {@link DefaultConnectionFactory#setMinBytes(int)}, + * controls latency and throughput. + * Smaller intervals increase responsiveness, but may increase + * the number of poll operations, potentially increasing CPU + * consumption and reducing throughput. + * @param maxWait timeout to wait + */ + public void setMaxWait(int maxWait) { + this.maxWait = maxWait; + } + + public String getClientId() { + return clientId; + } + + /** + * A client name to be used throughout this connection. + * @param clientId the client name + */ + public void setClientId(String clientId) { + this.clientId = clientId; + } + + /** + * The buffer size for this client + * @param bufferSize the buffer size + */ + public void setBufferSize(int bufferSize) { + this.bufferSize = bufferSize; + } + + /** + * The socket timeout for this client + * @param socketTimeout the socket timeout + */ + public void setSocketTimeout(int socketTimeout) { + this.socketTimeout = socketTimeout; + } + + /** + * The timeout on fetching metadata (e.g. partition leaders) + * @param fetchMetadataTimeout timeout + */ + public void setFetchMetadataTimeout(int fetchMetadataTimeout) { + this.fetchMetadataTimeout = fetchMetadataTimeout; + } + + /** + * @see ConnectionFactory#getLeaders(Iterable) + */ + @Override + public Map getLeaders(Iterable partitions) { + return FastList.newList(partitions).toMap(Functions.getPassThru(), getBrokersByPartitionFunction); + } + + /** + * @see ConnectionFactory#getLeader(Partition) + */ + @Override + public BrokerAddress getLeader(Partition partition) { + try { + lock.readLock().lock(); + return this.getLeaders(Collections.singleton(partition)).get(partition); + } + finally { + lock.readLock().unlock(); + } + } + + /** + * @see ConnectionFactory#connect(BrokerAddress) + */ + @Override + public Connection connect(BrokerAddress brokerAddress) { + return kafkaBrokersCache.getIfAbsentPutWithKey(brokerAddress, connectionInstantiationFunction); + } + + /** + * @see ConnectionFactory#refreshLeaders(Collection) + */ + @Override + public void refreshLeaders(Collection topics) { + try { + lock.writeLock().lock(); + for (Connection connection : kafkaBrokersCache) { + connection.close(); + } + String brokerAddressesAsString = + ListIterate.collect(configuration.getBrokerAddresses(), Functions.getToString()) + .makeString(","); + TopicMetadataResponse topicMetadataResponse = + new TopicMetadataResponse( + ClientUtils$.MODULE$.fetchTopicMetadata( + JavaConversions.asScalaSet(new HashSet(topics)), + ClientUtils$.MODULE$.parseBrokerList(brokerAddressesAsString), + getClientId(), fetchMetadataTimeout, 0)); + Map kafkaBrokerAddressMap = new HashMap(); + for (TopicMetadata topicMetadata : topicMetadataResponse.topicsMetadata()) { + for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) { + kafkaBrokerAddressMap.put(new Partition(topicMetadata.topic(), partitionMetadata.partitionId()), + new BrokerAddress(partitionMetadata.leader().host(), partitionMetadata.leader().port())); + } + } + this.partitionBrokerMapReference.set(new PartitionBrokerMap(UnifiedMap.newMap(kafkaBrokerAddressMap))); + } + finally { + lock.writeLock().unlock(); + } + } + + /** + * @see ConnectionFactory#getPartitions(String) + */ + @Override + public Collection getPartitions(String topic) { + if (!getPartitionBrokerMap().getPartitionsByTopic().containsKey(topic)) { + throw new TopicNotFoundException(topic); + } + return getPartitionBrokerMap().getPartitionsByTopic().get(topic).toList(); + } + + private PartitionBrokerMap getPartitionBrokerMap() { + return partitionBrokerMapReference.get(); + } + + @SuppressWarnings("serial") + private class ConnectionInstantiationFunction implements Function { + + @Override + public Connection valueOf(BrokerAddress brokerAddress) { + return new DefaultConnection(brokerAddress, clientId, bufferSize, socketTimeout, minBytes, maxWait); + } + + } + + @SuppressWarnings("serial") + private class GetBrokersByPartitionFunction implements Function { + + @Override + public BrokerAddress valueOf(Partition partition) { + return partitionBrokerMapReference.get().getBrokersByPartition().get(partition); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/FetchRequest.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/FetchRequest.java new file mode 100644 index 0000000000..a3b523f9e6 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/FetchRequest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +/** + * Encapsulates a request for fetching messages from the server. + * + * @author Marius Bogoevici + */ +public class FetchRequest { + + private Partition partition; + + private long offset; + + private int maxSizeInBytes; + + public FetchRequest(Partition Partition, long offset, int maxSizeInBytes) { + this.partition = Partition; + this.offset = offset; + this.maxSizeInBytes = maxSizeInBytes; + } + + public Partition getPartition() { + return partition; + } + + public void setPartition(Partition partition) { + this.partition = partition; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } + + public int getMaxSizeInBytes() { + return maxSizeInBytes; + } + + public void setMaxSizeInBytes(int maxSizeInBytes) { + this.maxSizeInBytes = maxSizeInBytes; + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java index e258f90cd2..2b332b7ca4 100644 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2013 the original author or authors. + * Copyright 2002-2015 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,29 +13,56 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.springframework.integration.kafka.core; +import kafka.api.OffsetRequest; + /** * Kafka adapter specific message headers. * * @author Soby Chacko + * @author Marius Bogoevici * @since 0.5 */ -public final class KafkaConsumerDefaults { +public abstract class KafkaConsumerDefaults { + //High level consumer public static final String GROUP_ID = "groupid"; - public static final String SOCKET_TIMEOUT = "30000"; - public static final String SOCKET_BUFFER_SIZE = "64*1024"; - public static final String FETCH_SIZE = "300 * 1024"; + + public static final int SOCKET_TIMEOUT_INT = 30000; + + public static final String SOCKET_TIMEOUT = Integer.toString(SOCKET_TIMEOUT_INT); + + public static final int SOCKET_BUFFER_SIZE_INT = 64*1024; + + public static final String SOCKET_BUFFER_SIZE = Integer.toString(SOCKET_BUFFER_SIZE_INT); + + public static final int FETCH_SIZE_INT = 300 * 1024; + + public static final String FETCH_SIZE = Integer.toString(FETCH_SIZE_INT); + public static final String BACKOFF_INCREMENT = "1000"; + public static final String QUEUED_CHUNKS_MAX = "100"; + public static final String AUTO_COMMIT_ENABLE = "true"; + public static final String AUTO_COMMIT_INTERVAL = "10000"; + public static final String AUTO_OFFSET_RESET = "smallest"; + //Overriding the default value of -1, which will make the consumer to wait indefinitely public static final String CONSUMER_TIMEOUT = "5000"; + public static final String REBALANCE_RETRIES_MAX = "4"; - private KafkaConsumerDefaults() { - } + public static final int MIN_FETCH_BYTES = 1; + + public static final int MAX_WAIT_TIME_IN_MS = 100; + + public static final long DEFAULT_OFFSET_RESET = OffsetRequest.EarliestTime(); + + public static final int FETCH_METADATA_TIMEOUT = 10000; + } diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessage.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessage.java new file mode 100644 index 0000000000..cbf8d19851 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessage.java @@ -0,0 +1,53 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import kafka.message.Message; + +import org.springframework.util.ObjectUtils; + +/** + * Wrapper around a {@link Message kafka message} and its {@link KafkaMessageMetadata metadata}. + * + * @author Marius Bogoevici + */ +public class KafkaMessage { + + private final Message message; + + private final KafkaMessageMetadata metadata; + + public KafkaMessage(Message message, KafkaMessageMetadata metadata) { + this.message = message; + this.metadata = metadata; + } + + public Message getMessage() { + return message; + } + + public KafkaMessageMetadata getMetadata() { + return metadata; + } + + @Override + public String toString() { + return "KafkaMessage [" + ObjectUtils.nullSafeToString(message) + ", " + + ObjectUtils.nullSafeToString(metadata.toString()) + "]"; + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageBatch.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageBatch.java new file mode 100644 index 0000000000..8e718cf0b0 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageBatch.java @@ -0,0 +1,64 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import java.util.List; + +/** + * A group of {@link KafkaMessage}s retrieved by a fetch operation + * + * @author Marius Bogoevici + */ +public class KafkaMessageBatch { + + private Partition partition; + + private List messages; + + private long highWatermark; + + public KafkaMessageBatch(Partition partition, List messages, long highWatermark) { + this.partition = partition; + this.messages = messages; + this.highWatermark = highWatermark; + } + + public Partition getPartition() { + return partition; + } + + public void setPartition(Partition partition) { + this.partition = partition; + } + + public List getMessages() { + return messages; + } + + public void setMessages(List messages) { + this.messages = messages; + } + + public long getHighWatermark() { + return highWatermark; + } + + public void setHighWatermark(long highWatermark) { + this.highWatermark = highWatermark; + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageMetadata.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageMetadata.java new file mode 100644 index 0000000000..aac33c6050 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageMetadata.java @@ -0,0 +1,65 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import kafka.message.Message; + +import org.springframework.util.Assert; + +/** + * Metadata for a Kafka {@link Message}. + * + * @author Marius Bogoevici + */ +public class KafkaMessageMetadata { + + private final long offset; + + private final long nextOffset; + + private final Partition partition; + + public KafkaMessageMetadata(Partition partition, long offset, long nextOffset) { + Assert.notNull(partition); + Assert.isTrue(offset >= 0); + Assert.isTrue(nextOffset >= 0); + this.offset = offset; + this.nextOffset = nextOffset; + this.partition = partition; + } + + public Partition getPartition() { + return partition; + } + + public long getOffset() { + return offset; + } + + public long getNextOffset() { + return nextOffset; + } + + @Override + public String toString() { + return "KafkaMessageMetadata [" + + "offset=" + offset + + ", nextOffset=" + nextOffset + + ", " + partition.toString(); + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaOperations.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaOperations.java new file mode 100644 index 0000000000..4d6a198c35 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaOperations.java @@ -0,0 +1,31 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +/** + * @author Marius Bogoevici + */ +public interface KafkaOperations { + + /** + * Receive data from the server. + * @param messageFetchRequests the set of FetchRequest + * @return the Result with KafkaMessageBatch + */ + Result receive(Iterable messageFetchRequests); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaTemplate.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaTemplate.java new file mode 100644 index 0000000000..659abb1a87 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaTemplate.java @@ -0,0 +1,65 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.list.MutableList; +import com.gs.collections.impl.list.mutable.FastList; + + +/** + * A template for executing high-level operations on a set of Kafka brokers. + * + * @author Marius Bogoevici + */ +public class KafkaTemplate implements KafkaOperations { + + private final ConnectionFactory connectionFactory; + + private FetchRequestToLeaderBrokerAddress fetchRequestToLeaderBrokerAddress = new FetchRequestToLeaderBrokerAddress(); + + public KafkaTemplate(ConnectionFactory connectionFactory) { + this.connectionFactory = connectionFactory; + } + + public ConnectionFactory getConnectionFactory() { + return connectionFactory; + } + + @Override + public Result receive(Iterable messageFetchRequests) { + FastList requestList = FastList.newList(messageFetchRequests); + MutableList distinctBrokerAddresses = + requestList.collect(fetchRequestToLeaderBrokerAddress).distinct(); + if (distinctBrokerAddresses.size() != 1) { + throw new IllegalArgumentException("All messages must be fetched from the same broker"); + } + return connectionFactory.connect(distinctBrokerAddresses.getFirst()) + .fetch(requestList.toTypedArray(FetchRequest.class)); + } + + @SuppressWarnings("serial") + private class FetchRequestToLeaderBrokerAddress implements Function { + + @Override + public BrokerAddress valueOf(FetchRequest fetchRequest) { + return connectionFactory.getLeader(fetchRequest.getPartition()); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Partition.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Partition.java new file mode 100644 index 0000000000..db19f4b3c7 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Partition.java @@ -0,0 +1,81 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +/** + * A reference to a Kafka partition, with both topic and partition id + * + * @author Marius Bogoevici + */ +public class Partition { + + private String topic; + + private int id; + + public Partition(String topic, int id) { + this.topic = topic; + this.id = id; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + @Override + public int hashCode() { + int result = topic.hashCode(); + result = 31 * result + id; + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Partition partition = (Partition) o; + if (id != partition.id) { + return false; + } + if (!topic.equals(partition.topic)) { + return false; + } + return true; + } + + @Override + public String toString() { + return "Partition[" + "topic='" + topic + '\'' + ", id=" + id + ']'; + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/PartitionBrokerMap.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/PartitionBrokerMap.java new file mode 100644 index 0000000000..6a7c0276a3 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/PartitionBrokerMap.java @@ -0,0 +1,66 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.map.ImmutableMap; +import com.gs.collections.api.multimap.Multimap; +import com.gs.collections.impl.map.mutable.UnifiedMap; + +/** + * Immutable store for the partition/broker mapping + * + * @author Marius Bogoevici + */ +class PartitionBrokerMap { + + private static final GetTopicFunction getTopicFunction = new GetTopicFunction(); + + private final Multimap partitionsByBroker; + + private final ImmutableMap brokersByPartition; + + private final Multimap partitionsByTopic; + + public PartitionBrokerMap(UnifiedMap brokersByPartition) { + this.brokersByPartition = brokersByPartition.toImmutable(); + this.partitionsByTopic = this.brokersByPartition.keysView().groupBy(getTopicFunction); + this.partitionsByBroker = brokersByPartition.flip().toImmutable(); + } + + public Multimap getPartitionsByBroker() { + return partitionsByBroker; + } + + public ImmutableMap getBrokersByPartition() { + return brokersByPartition; + } + + public Multimap getPartitionsByTopic() { + return partitionsByTopic; + } + + @SuppressWarnings("serial") + private static class GetTopicFunction implements Function { + + @Override + public String valueOf(Partition partition) { + return partition.getTopic(); + } + + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Result.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Result.java new file mode 100644 index 0000000000..a2544a95f0 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Result.java @@ -0,0 +1,64 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import java.util.Collections; +import java.util.Map; + +/** + * The result of a Kafka SimpleConsumer operation + * + * @author Marius Bogoevici + */ +public class Result { + + private final Map results; + + private final Map errors; + + Result(Map results, Map errors) { + this.results = Collections.unmodifiableMap(results); + this.errors = Collections.unmodifiableMap(errors); + } + + public Map getResults() { + return results; + } + + public T getResult(Partition partition) throws IllegalArgumentException { + if (this.results.containsKey(partition)) { + return this.results.get(partition); + } + else { + throw new IllegalArgumentException(" No result received for " + partition.toString()); + } + } + + public Map getErrors() { + return errors; + } + + public short getError(Partition partition) throws IllegalArgumentException { + if (this.getErrors().containsKey(partition)) { + return this.getErrors().get(partition); + } + else { + throw new IllegalArgumentException("No error received for " + partition.toString()); + } + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ResultBuilder.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ResultBuilder.java new file mode 100644 index 0000000000..fd2220d3ad --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ResultBuilder.java @@ -0,0 +1,74 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import java.util.HashMap; +import java.util.Map; + +/** + * Utility class for building a {@link Result} + * + * @author Marius Bogoevici + */ +class ResultBuilder { + + private Map result; + + private Map errors; + + public ResultBuilder() { + this.result = new HashMap(); + this.errors = new HashMap(); + } + + public KafkaPartitionResultHolder add(Partition Partition) { + return new KafkaPartitionResultHolder(Partition); + } + + public Result build() { + return new Result(result, errors); + } + + class KafkaPartitionResultHolder { + + private Partition partition; + + public KafkaPartitionResultHolder(Partition Partition) { + this.partition = Partition; + } + + public ResultBuilder withResult(T result) { + if (ResultBuilder.this.errors.containsKey(partition)) { + throw new IllegalArgumentException("A KafkaResult cannot contain both an error " + + "and a result for the same topic and partition"); + } + ResultBuilder.this.result.put(partition, result); + return ResultBuilder.this; + } + + public ResultBuilder withError(short error) { + if (ResultBuilder.this.result.containsKey(partition)) { + throw new IllegalArgumentException("A FetchResult cannot contain both an error " + + "and a MessageSet for the same topic and partition"); + } + ResultBuilder.this.errors.put(partition, error); + return ResultBuilder.this; + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/TopicNotFoundException.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/TopicNotFoundException.java new file mode 100644 index 0000000000..2d99c0c2cf --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/TopicNotFoundException.java @@ -0,0 +1,29 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.core; + +/** + * @author Marius Bogoevici + */ +@SuppressWarnings("serial") +public class TopicNotFoundException extends ConsumerException { + + public TopicNotFoundException(String topicName) { + super(String.format("No topic named '%s' found", topicName)); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConfiguration.java new file mode 100644 index 0000000000..5acca80c33 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConfiguration.java @@ -0,0 +1,99 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.core; + +import java.util.List; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.impl.list.mutable.FastList; +import kafka.cluster.Broker; +import kafka.utils.ZKStringSerializer$; +import kafka.utils.ZkUtils$; +import org.I0Itec.zkclient.ZkClient; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import scala.collection.JavaConversions; +import scala.collection.Seq; + +import org.springframework.beans.factory.BeanInitializationException; +import org.springframework.integration.kafka.support.ZookeeperConnect; + +/** + * Kafka {@link Configuration} that uses a ZooKeeper connection for retrieving the list of seed brokers. + * + * @author Marius Bogoevici + */ +public class ZookeeperConfiguration extends AbstractConfiguration { + + private final static Log log = LogFactory.getLog(ZookeeperConfiguration.class); + + public static final BrokerToBrokerAddressFunction brokerToBrokerAddressFunction = new BrokerToBrokerAddressFunction(); + + private String zookeeperServers; + + private int sessionTimeout; + + private int connectionTimeout; + + public ZookeeperConfiguration(ZookeeperConnect zookeeperConnect) { + this.zookeeperServers = zookeeperConnect.getZkConnect(); + try { + this.sessionTimeout = Integer.parseInt(zookeeperConnect.getZkSessionTimeout()); + } + catch (NumberFormatException e) { + throw new BeanInitializationException("Cannot parse session timeout:", e); + } + try { + this.connectionTimeout = Integer.parseInt(zookeeperConnect.getZkConnectionTimeout()); + } + catch (NumberFormatException e) { + throw new BeanInitializationException("Cannot parse connection timeout:", e); + } + } + + @Override + protected List doGetBrokerAddresses() { + ZkClient zkClient = null; + try { + zkClient = new ZkClient(zookeeperServers, sessionTimeout, connectionTimeout, ZKStringSerializer$.MODULE$); + Seq allBrokersInCluster = ZkUtils$.MODULE$.getAllBrokersInCluster(zkClient); + FastList brokers = FastList.newList(JavaConversions.asJavaCollection(allBrokersInCluster)); + return brokers.collect(brokerToBrokerAddressFunction); + } + finally { + if (zkClient != null) { + try { + zkClient.close(); + } + catch (Exception e) { + log.error("Cannot close Zookeeper client: ", e); + } + } + } + } + + @SuppressWarnings("serial") + private static class BrokerToBrokerAddressFunction implements Function { + + @Override + public BrokerAddress valueOf(Broker broker) { + return new BrokerAddress(broker.host(), broker.port()); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaMessageDrivenChannelAdapter.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaMessageDrivenChannelAdapter.java new file mode 100644 index 0000000000..6115ae25d0 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaMessageDrivenChannelAdapter.java @@ -0,0 +1,112 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.inbound; + +import kafka.serializer.Decoder; +import kafka.serializer.DefaultDecoder; + +import org.springframework.integration.context.OrderlyShutdownCapable; +import org.springframework.integration.endpoint.MessageProducerSupport; +import org.springframework.integration.kafka.core.KafkaMessageMetadata; +import org.springframework.integration.kafka.listener.AbstractDecodingMessageListener; +import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer; +import org.springframework.integration.kafka.support.KafkaHeaders; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.util.Assert; + +/** + * @author Marius Bogoevici + */ +public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport implements OrderlyShutdownCapable { + + private KafkaMessageListenerContainer messageListenerContainer; + + private Decoder keyDecoder = new DefaultDecoder(null); + + private Decoder payloadDecoder = new DefaultDecoder(null); + + public KafkaMessageDrivenChannelAdapter(KafkaMessageListenerContainer messageListenerContainer) { + Assert.notNull(messageListenerContainer); + Assert.isNull(messageListenerContainer.getMessageListener()); + this.messageListenerContainer = messageListenerContainer; + this.messageListenerContainer.setAutoStartup(false); + } + + public void setKeyDecoder(Decoder keyDecoder) { + this.keyDecoder = keyDecoder; + } + + public void setPayloadDecoder(Decoder payloadDecoder) { + this.payloadDecoder = payloadDecoder; + } + + @Override + protected void onInit() { + this.messageListenerContainer.setMessageListener(new ChannelForwardingMessageListener()); + super.onInit(); + } + + @Override + protected void doStart() { + this.messageListenerContainer.start(); + } + + @Override + protected void doStop() { + this.messageListenerContainer.stop(); + } + + @Override + public String getComponentType() { + return "kafka:message-driven-channel-adapter"; + } + + @Override + public int beforeShutdown() { + this.messageListenerContainer.stop(); + return getPhase(); + } + + @Override + public int afterShutdown() { + return getPhase(); + } + + @SuppressWarnings("rawtypes") + private class ChannelForwardingMessageListener extends AbstractDecodingMessageListener { + + @SuppressWarnings("unchecked") + public ChannelForwardingMessageListener() { + super(keyDecoder, payloadDecoder); + } + + @Override + public void doOnMessage(Object key, Object payload, KafkaMessageMetadata metadata) { + Message message = getMessageBuilderFactory() + .withPayload(payload) + .setHeader(KafkaHeaders.MESSAGE_KEY, key) + .setHeader(KafkaHeaders.TOPIC, metadata.getPartition().getTopic()) + .setHeader(KafkaHeaders.PARTITION_ID, metadata.getPartition().getId()) + .setHeader(KafkaHeaders.OFFSET, metadata.getOffset()) + .build(); + KafkaMessageDrivenChannelAdapter.this.sendMessage(message); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingMessageListener.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingMessageListener.java new file mode 100644 index 0000000000..9db10afa38 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingMessageListener.java @@ -0,0 +1,62 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.listener; + + +import kafka.serializer.Decoder; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.KafkaMessageMetadata; +import org.springframework.integration.kafka.util.MessageUtils; +import org.springframework.util.Assert; + +/** + * Base {@link MessageListener} implementation that decodes the key and the payload using the supplied + * {@link Decoder}s. + * + * Users of this class must extend it and implement {@code doOnMessage} and must supply {@link Decoder} + * implementations for both the key and the payload. + * + * @author Marius Bogoevici + */ +public abstract class AbstractDecodingMessageListener implements MessageListener { + + private Decoder keyDecoder; + + private Decoder

payloadDecoder; + + public AbstractDecodingMessageListener(Decoder keyDecoder, Decoder

payloadDecoder) { + this.keyDecoder = keyDecoder; + this.payloadDecoder = payloadDecoder; + } + + @Override + public final void onMessage(KafkaMessage message) { + this.doOnMessage(MessageUtils.decodeKey(message, keyDecoder), + MessageUtils.decodePayload(message, payloadDecoder), message.getMetadata()); + } + + /** + * Process the decoded message + * @param key the message key + * @param payload the message body + * @param metadata the KafkaMessageMetadata + */ + public abstract void doOnMessage(K key, P payload, KafkaMessageMetadata metadata); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ConcurrentMessageListenerDispatcher.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ConcurrentMessageListenerDispatcher.java new file mode 100644 index 0000000000..62bbffe4a1 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ConcurrentMessageListenerDispatcher.java @@ -0,0 +1,157 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.listener; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +import com.gs.collections.api.block.procedure.Procedure; +import com.gs.collections.api.block.procedure.Procedure2; +import com.gs.collections.api.map.MutableMap; +import com.gs.collections.impl.factory.Maps; + +import org.springframework.context.Lifecycle; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.scheduling.concurrent.CustomizableThreadFactory; +import org.springframework.util.Assert; + +/** + * Dispatches {@link KafkaMessage}s to a {@link MessageListener}. Messages may be + * processed concurrently, according to the {@code concurrency} settings, but messages + * from the same partition are being processed in their original order. + * + * @author Marius Bogoevici + */ +class ConcurrentMessageListenerDispatcher implements Lifecycle { + + public static final CustomizableThreadFactory THREAD_FACTORY = new CustomizableThreadFactory("dispatcher-"); + + private static final StartDelegateProcedure startDelegateProcedure = new StartDelegateProcedure(); + + private static final StopDelegateProcedure stopDelegateProcedure = new StopDelegateProcedure(); + + private final Object lifecycleMonitor = new Object(); + + private final Collection partitions; + + private final int consumers; + + private volatile boolean running; + + private final MessageListener delegateListener; + + private final ErrorHandler errorHandler; + + private final OffsetManager offsetManager; + + private MutableMap delegates; + + private final int queueSize; + + private Executor taskExecutor; + + public ConcurrentMessageListenerDispatcher(MessageListener delegateListener, ErrorHandler errorHandler, + Collection partitions, OffsetManager offsetManager, int consumers, int queueSize) { + Assert.notEmpty(partitions, "A set of partitions must be provided"); + Assert.isTrue(consumers <= partitions.size(), + "The number of consumers must be smaller or equal to the number of partitions"); + Assert.notNull(delegateListener, "A delegate must be provided"); + this.delegateListener = delegateListener; + this.errorHandler = errorHandler; + this.partitions = partitions; + this.offsetManager = offsetManager; + this.consumers = consumers; + this.queueSize = queueSize; + } + + @Override + public void start() { + synchronized (lifecycleMonitor) { + if (!isRunning()) { + initializeAndStartDispatching(); + this.running = true; + } + } + } + + @Override + public void stop() { + synchronized (lifecycleMonitor) { + if (isRunning()) { + this.running = false; + delegates.flip().keyBag().toSet().forEach(stopDelegateProcedure); + } + } + } + + @Override + public boolean isRunning() { + return running; + } + + public void dispatch(KafkaMessage message) { + delegates.get(message.getMetadata().getPartition()).enqueue(message); + } + + private void initializeAndStartDispatching() { + // allocate delegate instances index them + List delegateList = new ArrayList(consumers); + for (int i = 0; i < consumers; i++) { + QueueingMessageListenerInvoker blockingQueueMessageListenerInvoker = + new QueueingMessageListenerInvoker(queueSize, offsetManager, delegateListener, errorHandler); + delegateList.add(blockingQueueMessageListenerInvoker); + } + // evenly distribute partitions across delegates + delegates = Maps.mutable.of(); + int i = 0; + for (Partition partition : partitions) { + delegates.put(partition, delegateList.get((i++) % consumers)); + } + // initialize task executor + if (this.taskExecutor == null) { + this.taskExecutor = Executors.newFixedThreadPool(consumers, THREAD_FACTORY); + } + // start dispatchers + delegates.flip().keyBag().toSet().forEachWith(startDelegateProcedure, taskExecutor); + } + + @SuppressWarnings("serial") + private static class StopDelegateProcedure implements Procedure { + + @Override + public void value(QueueingMessageListenerInvoker delegate) { + delegate.stop(); + } + + } + + @SuppressWarnings("serial") + private static class StartDelegateProcedure implements Procedure2 { + + @Override + public void value(QueueingMessageListenerInvoker delegate, Executor executor) { + delegate.start(); + executor.execute(delegate); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ErrorHandler.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ErrorHandler.java new file mode 100644 index 0000000000..259389375b --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ErrorHandler.java @@ -0,0 +1,31 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.springframework.integration.kafka.core.KafkaMessage; + +/** + * Handles errors thrown during the execution of a {@link MessageListener} + * + * @author Marius Bogoevici + */ +public interface ErrorHandler { + + void handle(Exception thrownException, KafkaMessage message); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaMessageListenerContainer.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaMessageListenerContainer.java new file mode 100644 index 0000000000..9337f8d73d --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaMessageListenerContainer.java @@ -0,0 +1,578 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.listener; + +import static com.gs.collections.impl.utility.ArrayIterate.flatCollect; +import static com.gs.collections.impl.utility.Iterate.partition; +import static com.gs.collections.impl.utility.MapIterate.forEachKeyValue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +import com.gs.collections.api.RichIterable; +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.block.predicate.Predicate; +import com.gs.collections.api.block.procedure.Procedure; +import com.gs.collections.api.block.procedure.Procedure2; +import com.gs.collections.api.collection.MutableCollection; +import com.gs.collections.api.list.ImmutableList; +import com.gs.collections.api.list.MutableList; +import com.gs.collections.api.multimap.MutableMultimap; +import com.gs.collections.api.partition.PartitionIterable; +import com.gs.collections.impl.block.factory.Functions; +import com.gs.collections.impl.block.function.checked.CheckedFunction; +import com.gs.collections.impl.factory.Lists; +import com.gs.collections.impl.factory.Multimaps; +import com.gs.collections.impl.list.mutable.FastList; + +import kafka.common.ErrorMapping; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.context.SmartLifecycle; +import org.springframework.integration.kafka.core.BrokerAddress; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.ConsumerException; +import org.springframework.integration.kafka.core.FetchRequest; +import org.springframework.integration.kafka.core.KafkaConsumerDefaults; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.KafkaMessageBatch; +import org.springframework.integration.kafka.core.KafkaTemplate; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.Result; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +/** + * @author Marius Bogoevici + */ +public class KafkaMessageListenerContainer implements SmartLifecycle { + + private static final Log log = LogFactory.getLog(KafkaMessageListenerContainer.class); + + public static final Function, Partition> keyFunction = Functions.getKeyFunction(); + + private final GetOffsetForPartitionFunction getOffset = new GetOffsetForPartitionFunction(); + + private final PartitionToLeaderFunction getLeader = new PartitionToLeaderFunction(); + + private final Function passThru = Functions.getPassThru(); + + private final LaunchFetchTaskProcedure launchFetchTask = new LaunchFetchTaskProcedure(); + + private final Object lifecycleMonitor = new Object(); + + private final KafkaTemplate kafkaTemplate; + + private Partition[] partitions; + + private String[] topics; + + public boolean autoStartup = true; + + private Executor fetchTaskExecutor; + + private Executor adminTaskExecutor = Executors.newSingleThreadExecutor(); + + private int concurrency = 1; + + private volatile boolean running = false; + + private int maxFetch = KafkaConsumerDefaults.FETCH_SIZE_INT; + + private int queueSize = 1024; + + private MessageListener messageListener; + + private ErrorHandler errorHandler = new LoggingErrorHandler(); + + private volatile OffsetManager offsetManager; + + private ConcurrentMap fetchOffsets; + + private ConcurrentMessageListenerDispatcher messageDispatcher; + + private final MutableMultimap partitionsByBrokerMap = Multimaps.mutable.set.with(); + + public KafkaMessageListenerContainer(ConnectionFactory connectionFactory, Partition... partitions) { + Assert.notNull(connectionFactory, "A connection factory must be supplied"); + Assert.notEmpty(partitions, "A list of partitions must be provided"); + Assert.noNullElements(partitions, "The list of partitions cannot contain null elements"); + this.kafkaTemplate = new KafkaTemplate(connectionFactory); + this.partitions = partitions; + } + + public KafkaMessageListenerContainer(final ConnectionFactory connectionFactory, String... topics) { + Assert.notNull(connectionFactory, "A connection factory must be supplied"); + Assert.notNull(topics, "A list of topics must be provided"); + Assert.noNullElements(topics, "The list of topics cannot contain null elements"); + this.kafkaTemplate = new KafkaTemplate(connectionFactory); + this.topics = topics; + } + + private static Partition[] getPartitionsForTopics(final ConnectionFactory connectionFactory, String[] topics) { + MutableList partitionList = flatCollect(topics, new GetPartitionsForTopic(connectionFactory)); + return partitionList.toArray(new Partition[partitionList.size()]); + } + + public OffsetManager getOffsetManager() { + return offsetManager; + } + + public void setOffsetManager(OffsetManager offsetManager) { + this.offsetManager = offsetManager; + } + + public MessageListener getMessageListener() { + return messageListener; + } + + public void setMessageListener(MessageListener messageListener) { + this.messageListener = messageListener; + } + + public ErrorHandler getErrorHandler() { + return errorHandler; + } + + public void setErrorHandler(ErrorHandler errorHandler) { + this.errorHandler = errorHandler; + } + + public int getConcurrency() { + return concurrency; + } + + /** + * The maximum number of concurrent {@link MessageListener}s running. Messages from within the same + * partition will be processed sequentially. + * @param concurrency the concurrency maximum number + */ + public void setConcurrency(int concurrency) { + this.concurrency = concurrency; + } + + public Executor getFetchTaskExecutor() { + return fetchTaskExecutor; + } + + /** + * The task executor for fetch operations + * @param fetchTaskExecutor the Executor for fetch operations + */ + public void setFetchTaskExecutor(Executor fetchTaskExecutor) { + this.fetchTaskExecutor = fetchTaskExecutor; + } + + /** + * @return the task executor for leader and offset updates. + */ + public Executor getAdminTaskExecutor() { + return adminTaskExecutor; + } + + public void setAdminTaskExecutor(Executor adminTaskExecutor) { + this.adminTaskExecutor = adminTaskExecutor; + } + + /** + * @return the maximum amount of data (in bytes) that pollers will fetch in one round + */ + public int getMaxFetch() { + return maxFetch; + } + + public int getQueueSize() { + return queueSize; + } + + /** + * The maximum number of messages that are buffered by each concurrent {@link MessageListener} runner. + * Increasing the value may increase throughput, but also increases the memory consumption. + * @param queueSize the queue size + */ + public void setQueueSize(int queueSize) { + this.queueSize = queueSize; + } + + public void setMaxFetch(int maxFetch) { + this.maxFetch = maxFetch; + } + + @Override + public boolean isAutoStartup() { + return autoStartup; + } + + public void setAutoStartup(boolean autoStartup) { + this.autoStartup = autoStartup; + } + + @Override + public void stop(Runnable callback) { + synchronized (lifecycleMonitor) { + if (running) { + this.running = false; + try { + this.offsetManager.flush(); + } + catch (IOException e) { + log.error("Error while flushing:", e); + } + try { + this.offsetManager.close(); + } + catch (IOException e) { + log.error("Error while closing:", e); + } + this.messageDispatcher.stop(); + } + } + if (callback != null) { + callback.run(); + } + } + + @Override + public void start() { + synchronized (lifecycleMonitor) { + if (!running) { + if (partitions == null) { + partitions = getPartitionsForTopics(kafkaTemplate.getConnectionFactory(), topics); + } + this.running = true; + if (this.offsetManager == null) { + this.offsetManager = new MetadataStoreOffsetManager(kafkaTemplate.getConnectionFactory()); + } + // initialize the fetch offset table - defer to OffsetManager for retrieving them + ImmutableList partitionsAsList = Lists.immutable.with(partitions); + this.fetchOffsets = new ConcurrentHashMap(partitionsAsList.toMap(passThru, getOffset)); + this.messageDispatcher = new ConcurrentMessageListenerDispatcher(messageListener, errorHandler, + Arrays.asList(partitions), offsetManager, concurrency, queueSize); + this.messageDispatcher.start(); + partitionsByBrokerMap.putAll(partitionsAsList.groupBy(getLeader)); + if (fetchTaskExecutor == null) { + fetchTaskExecutor = Executors.newFixedThreadPool(partitionsByBrokerMap.size()); + } + partitionsByBrokerMap.forEachKey(launchFetchTask); + } + } + } + + @Override + public void stop() { + this.stop(null); + } + + @Override + public boolean isRunning() { + return this.running; + } + + @Override + public int getPhase() { + return 0; + } + + /** + * Fetches data from Kafka for a group of partitions, located on the same broker. + */ + public class FetchTask implements Runnable { + + private BrokerAddress brokerAddress; + + public FetchTask(BrokerAddress brokerAddress) { + this.brokerAddress = brokerAddress; + } + + @Override + public void run() { + boolean wasInterrupted = false; + while (isRunning()) { + MutableCollection fetchPartitions; + synchronized (partitionsByBrokerMap) { + // retrieve the partitions for the current polling cycle + fetchPartitions = partitionsByBrokerMap.get(brokerAddress); + // do not proceed until there is something to read from + while (isRunning() && CollectionUtils.isEmpty(fetchPartitions)) { + try { + // we only got here because there were no partitions to read from, + // so block until there is a change this prevents FetchTasks + // from busy waiting while leaders or offsets are being refreshed + // TODO: ideally we should use separate monitors for each task + partitionsByBrokerMap.wait(); + // see if the changes affect us + fetchPartitions = partitionsByBrokerMap.get(brokerAddress); + } + catch (InterruptedException e) { + wasInterrupted = true; + } + } + } + // we've just exited a potentially blocking operation. Is the component still running? + if (isRunning()) { + Set partitionsWithRemainingData; + boolean hasErrors; + do { + partitionsWithRemainingData = new HashSet(); + hasErrors = false; + try { + MutableCollection fetchRequests = + fetchPartitions.collect(new PartitionToFetchRequestFunction()); + Result result = kafkaTemplate.receive(fetchRequests); + // process successful messages first + Iterable batches = result.getResults().values(); + for (KafkaMessageBatch batch : batches) { + if (!batch.getMessages().isEmpty()) { + long highestFetchedOffset = 0; + for (KafkaMessage kafkaMessage : batch.getMessages()) { + // fetch operations may return entire blocks of compressed messages, + // which may have lower offsets than the ones requested + // thus a batch may contain messages that have been processed already + if (kafkaMessage.getMetadata().getOffset() >= fetchOffsets.get(batch.getPartition())) { + messageDispatcher.dispatch(kafkaMessage); + } + highestFetchedOffset = + Math.max(highestFetchedOffset, kafkaMessage.getMetadata().getNextOffset()); + } + fetchOffsets.replace(batch.getPartition(), highestFetchedOffset); + // if there are still messages on server, we can go on and retrieve more + if (highestFetchedOffset < batch.getHighWatermark()) { + partitionsWithRemainingData.add(batch.getPartition()); + } + } + } + // handle errors + if (result.getErrors().size() > 0) { + hasErrors = true; + + // find partitions with leader errors and + PartitionIterable> partitionByLeaderErrors = + partition(result.getErrors().entrySet(), new IsLeaderErrorPredicate()); + RichIterable partitionsWithLeaderErrors = + partitionByLeaderErrors.getSelected().collect(keyFunction); + resetLeaders(partitionsWithLeaderErrors); + + PartitionIterable> partitionsWithOffsetsOutOfRange = + partitionByLeaderErrors.getRejected() + .partition(new IsOffsetOutOfRangePredicate()); + resetOffsets(partitionsWithOffsetsOutOfRange.getSelected() + .collect(keyFunction) + .toSet()); + // it's not a leader issue + stopFetchingFromPartitions(partitionsWithOffsetsOutOfRange.getRejected() + .collect(keyFunction)); + } + } + catch (ConsumerException e) { + // this is a broker error, and we cannot recover from it. + // Reset leaders and stop fetching data from this broker altogether + log.error(e); + resetLeaders(fetchPartitions.toImmutable()); + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + return; + } + } while (!hasErrors && !partitionsWithRemainingData.isEmpty()); + } + } + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + } + + + private void resetLeaders(final Iterable partitionsToReset) { + stopFetchingFromPartitions(partitionsToReset); + adminTaskExecutor.execute(new UpdateLeadersTask(partitionsToReset)); + } + + + private void resetOffsets(final Collection partitionsToResetOffsets) { + stopFetchingFromPartitions(partitionsToResetOffsets); + adminTaskExecutor.execute(new UpdateOffsetsTask(partitionsToResetOffsets)); + } + + private void stopFetchingFromPartitions(Iterable partitions) { + synchronized (partitionsByBrokerMap) { + for (Partition partition : partitions) { + partitionsByBrokerMap.remove(brokerAddress, partition); + } + } + } + + private class UpdateLeadersTask implements Runnable { + private final Iterable partitionsToReset; + + public UpdateLeadersTask(Iterable partitionsToReset) { + this.partitionsToReset = partitionsToReset; + } + + @Override + public void run() { + FastList partitionsAsList = FastList.newList(partitionsToReset); + FastList topics = partitionsAsList.collect(new PartitionToTopicFunction()).distinct(); + kafkaTemplate.getConnectionFactory().refreshLeaders(topics); + Map leaders = kafkaTemplate.getConnectionFactory().getLeaders(partitionsToReset); + synchronized (partitionsByBrokerMap) { + forEachKeyValue(leaders, new AddPartitionToBrokerProcedure()); + partitionsByBrokerMap.notifyAll(); + } + } + + } + + private class UpdateOffsetsTask implements Runnable { + + private final Collection partitionsToResetOffsets; + + public UpdateOffsetsTask(Collection partitionsToResetOffsets) { + this.partitionsToResetOffsets = partitionsToResetOffsets; + } + + @Override + public void run() { + offsetManager.resetOffsets(partitionsToResetOffsets); + for (Partition partition : partitionsToResetOffsets) { + fetchOffsets.replace(partition, offsetManager.getOffset(partition)); + } + synchronized (partitionsByBrokerMap) { + for (Partition partitionsToResetOffset : partitionsToResetOffsets) { + partitionsByBrokerMap.put(brokerAddress, partitionsToResetOffset); + } + // notify any waiting task that the partition allocation has changed + partitionsByBrokerMap.notifyAll(); + } + } + + } + + @SuppressWarnings("serial") + private class IsLeaderErrorPredicate implements Predicate> { + + @Override + public boolean accept(Map.Entry each) { + return each.getValue() == ErrorMapping.NotLeaderForPartitionCode() + || each.getValue() == ErrorMapping.UnknownTopicOrPartitionCode(); + } + + } + + @SuppressWarnings("serial") + private class IsOffsetOutOfRangePredicate implements Predicate> { + + @Override + public boolean accept(Map.Entry each) { + return each.getValue() == ErrorMapping.OffsetOutOfRangeCode(); + } + + } + } + + @SuppressWarnings("serial") + class GetOffsetForPartitionFunction extends CheckedFunction { + + @Override + public Long safeValueOf(Partition object) throws Exception { + try { + return offsetManager.getOffset(object); + } + catch (Exception e) { + log.error(e); + throw e; + } + } + + } + + @SuppressWarnings("serial") + private class PartitionToLeaderFunction implements Function { + + @Override + public BrokerAddress valueOf(Partition partition) { + return kafkaTemplate.getConnectionFactory().getLeader(partition); + } + + } + + @SuppressWarnings("serial") + private class LaunchFetchTaskProcedure implements Procedure { + + @Override + public void value(BrokerAddress brokerAddress) { + fetchTaskExecutor.execute(new FetchTask(brokerAddress)); + } + + } + + @SuppressWarnings("serial") + private class PartitionToFetchRequestFunction implements Function { + + @Override + public FetchRequest valueOf(Partition partition) { + return new FetchRequest(partition, fetchOffsets.get(partition), maxFetch); + } + + } + + @SuppressWarnings("serial") + static class GetPartitionsForTopic extends CheckedFunction> { + + private final ConnectionFactory connectionFactory; + + public GetPartitionsForTopic(ConnectionFactory connectionFactory) { + this.connectionFactory = connectionFactory; + } + + @Override + public Iterable safeValueOf(String topic) throws Exception { + return connectionFactory.getPartitions(topic); + } + + } + + @SuppressWarnings("serial") + private class PartitionToTopicFunction implements Function { + + @Override + public String valueOf(Partition object) { + return object.getTopic(); + } + + } + + @SuppressWarnings("serial") + private class AddPartitionToBrokerProcedure implements Procedure2 { + + @Override + public void value(Partition partition, BrokerAddress newBrokerAddress) { + partitionsByBrokerMap.put(newBrokerAddress, partition); + } + + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LoggingErrorHandler.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LoggingErrorHandler.java new file mode 100644 index 0000000000..c149a354fb --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LoggingErrorHandler.java @@ -0,0 +1,38 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.util.ObjectUtils; + +/** + * @author Marius Bogoevici + */ +public class LoggingErrorHandler implements ErrorHandler { + + private static final Log log = LogFactory.getLog(LoggingErrorHandler.class); + + @Override + public void handle(Exception thrownException, KafkaMessage message) { + log.error("Error while processing: " + ObjectUtils.nullSafeToString(message), thrownException); + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MessageListener.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MessageListener.java new file mode 100644 index 0000000000..554df9efef --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MessageListener.java @@ -0,0 +1,35 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.springframework.integration.kafka.core.KafkaMessage; + +/** + * Listener for handling incoming Kafka messages + * + * @author Marius Bogoevici + */ +public interface MessageListener { + + /** + * Executes when a Kafka message is received + * @param message the Kafka message to be processed + */ + void onMessage(KafkaMessage message); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MetadataStoreOffsetManager.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MetadataStoreOffsetManager.java new file mode 100644 index 0000000000..92a3611ff2 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MetadataStoreOffsetManager.java @@ -0,0 +1,188 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.listener; + +import java.io.Closeable; +import java.io.Flushable; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import kafka.common.ErrorMapping; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.integration.kafka.core.BrokerAddress; +import org.springframework.integration.kafka.core.Connection; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.ConsumerException; +import org.springframework.integration.kafka.core.KafkaConsumerDefaults; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.Result; +import org.springframework.integration.metadata.MetadataStore; +import org.springframework.integration.metadata.SimpleMetadataStore; + +/** + * An {@link OffsetManager} that persists offsets into a {@link MetadataStore}. + * + * @author Marius Bogoevici + */ +public class MetadataStoreOffsetManager implements OffsetManager { + + private final static Log log = LogFactory.getLog(MetadataStoreOffsetManager.class); + + private String consumerId = KafkaConsumerDefaults.GROUP_ID; + + private MetadataStore metadataStore = new SimpleMetadataStore(); + + private ConnectionFactory connectionFactory; + + private Map initialOffsets; + + private long referenceTimestamp = KafkaConsumerDefaults.DEFAULT_OFFSET_RESET; + + public MetadataStoreOffsetManager(ConnectionFactory connectionFactory) { + this(connectionFactory, null); + } + + public MetadataStoreOffsetManager(ConnectionFactory connectionFactory, Map initialOffsets) { + this.connectionFactory = connectionFactory; + this.initialOffsets = initialOffsets == null? new HashMap() : initialOffsets; + } + + public String getConsumerId() { + return consumerId; + } + + /** + * The identifier of a consumer of Kafka messages. Allows to store separate sets of offsets in the + * {@link MetadataStore}. + * @param consumerId the consumer ID + */ + public void setConsumerId(String consumerId) { + this.consumerId = consumerId; + } + + public MetadataStore getMetadataStore() { + return metadataStore; + } + + /** + * The backing {@link MetadataStore} for storing offsets. + * @param metadataStore a fully configured {@link MetadataStore} instance + */ + public void setMetadataStore(MetadataStore metadataStore) { + this.metadataStore = metadataStore; + } + + public long getReferenceTimestamp() { + return referenceTimestamp; + } + + /** + * A timestamp to be used for resetting initial offsets, if they are not available in the {@link MetadataStore} + * @param referenceTimestamp the reset timestamp for initial offsets + */ + public void setReferenceTimestamp(long referenceTimestamp) { + this.referenceTimestamp = referenceTimestamp; + } + + /** + * @see OffsetManager#updateOffset(Partition, long) + */ + @Override + public synchronized void updateOffset(Partition partition, long offset) { + metadataStore.put(asKey(partition), Long.toString(offset)); + } + + /** + * @see OffsetManager#getOffset(Partition) + */ + @Override + public synchronized long getOffset(Partition partition) { + Long offsetInMetadataStore = getOffsetFromMetadataStore(partition); + if (offsetInMetadataStore == null) { + if (this.initialOffsets.containsKey(partition)) { + return this.initialOffsets.get(partition); + } + else { + BrokerAddress leader = this.connectionFactory.getLeader(partition); + if (leader == null) { + throw new ConsumerException("No leader found for " + partition.toString()); + } + Connection connection = this.connectionFactory.connect(leader); + Result offsetResult = connection.fetchInitialOffset(referenceTimestamp, partition); + if (offsetResult.getErrors().size() > 0) { + throw new ConsumerException(ErrorMapping.exceptionFor(offsetResult.getError(partition))); + } + if (!offsetResult.getResults().containsKey(partition)) { + throw new IllegalStateException("Result does not contain an expected value"); + } + return offsetResult.getResult(partition); + } + } + else { + return offsetInMetadataStore; + } + } + + @Override + public synchronized void resetOffsets(Collection partitionsToReset) { + // any information about those offsets is invalid and can be ignored + for (Partition partition : partitionsToReset) { + metadataStore.remove(asKey(partition)); + initialOffsets.remove(partition); + } + } + + @Override + public void close() throws IOException { + // Flush before closing. This may be redundant, but ensures that the metadata store is closed properly + flush(); + if (metadataStore instanceof Closeable) { + ((Closeable) metadataStore).close(); + } + } + + @Override + public void flush() throws IOException { + if (metadataStore instanceof Flushable) { + ((Flushable) metadataStore).flush(); + } + } + + + private Long getOffsetFromMetadataStore(Partition partition) { + String storedOffsetValueAsString = this.metadataStore.get(asKey(partition)); + Long storedOffsetValue = null; + if (storedOffsetValueAsString != null) { + try { + storedOffsetValue = Long.parseLong(storedOffsetValueAsString); + } + catch (NumberFormatException e) { + log.warn("Invalid value: " + storedOffsetValueAsString); + } + } + return storedOffsetValue; + } + + private String asKey(Partition partition) { + return partition.getTopic() + ":" + partition.getId() + ":" + consumerId; + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/OffsetManager.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/OffsetManager.java new file mode 100644 index 0000000000..b41207bcfc --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/OffsetManager.java @@ -0,0 +1,54 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.listener; + +import java.io.Closeable; +import java.io.Flushable; +import java.util.Collection; + +import org.springframework.integration.kafka.core.Partition; + +/** + * Stores and retrieves offsets for a Kafka consumer + * + * @author Marius Bogoevici + */ +public interface OffsetManager extends Closeable, Flushable { + + /** + * Updates the offset for a given {@link Partition} + * @param partition the partition whose offset is to be updated + * @param offset the new offset value + */ + void updateOffset(Partition partition, long offset); + + /** + * Retrieves the offset for a given {@link Partition} + * + * @param partition the partition to be + * @return the offset value + */ + long getOffset(Partition partition); + + /** + * Resets offsets for the given {@link Partition}s. To be invoked when the values stored are invalid, + * so a client cannot resume from that position. Implementations must decide on the best strategy to follow. + * @param partition to reset + */ + void resetOffsets(Collection partition); + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/QueueingMessageListenerInvoker.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/QueueingMessageListenerInvoker.java new file mode 100644 index 0000000000..8053b76f8d --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/QueueingMessageListenerInvoker.java @@ -0,0 +1,126 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.listener; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +import org.springframework.context.Lifecycle; +import org.springframework.integration.kafka.core.KafkaMessage; + +/** + * Invokes a delegate {@link MessageListener} for all the messages passed to it, storing them + * in an internal queue. + * + * @author Marius Bogoevici + */ +class QueueingMessageListenerInvoker implements Runnable, Lifecycle { + + private BlockingQueue messages; + + private volatile boolean running = false; + + private final MessageListener delegate; + + private final OffsetManager offsetManager; + + private final ErrorHandler errorHandler; + + public QueueingMessageListenerInvoker(int capacity, OffsetManager offsetManager, MessageListener delegate, + ErrorHandler errorHandler) { + this.offsetManager = offsetManager; + this.delegate = delegate; + this.errorHandler = errorHandler; + this.messages = new ArrayBlockingQueue(capacity, true); + } + + /** + * Add a message to the queue, blocking if the queue has reached its maximum capacity. + * Interrupts will be ignored for as long as the component's {@code running} flag is set to true, but will + * be deferred for when the method returns. + * @param message the KafkaMessage to add + */ + public void enqueue(KafkaMessage message) { + boolean wasInterruptedWhileRunning = false; + if (this.running) { + boolean added = false; + // handle the case when the thread is interrupted while the adapter is still running + // retry adding the message to the queue until either we succeed, or the adapter is stopped + while (!added && this.running) { + try { + this.messages.put(message); + added = true; + } + catch (InterruptedException e) { + // we ignore the interruption signal if we are still running, but pass it on if we are stopped + wasInterruptedWhileRunning = true; + } + } + } + if (wasInterruptedWhileRunning) { + Thread.currentThread().interrupt(); + } + } + + @Override + public void start() { + this.running = true; + } + + @Override + public void stop() { + this.running = false; + } + + @Override + public boolean isRunning() { + return this.running; + } + + /** + * Runs uninterruptibly as long as {@code running} is true, but if interrupted, will defer + * propagating the interruption flag at the end. + */ + @Override + public void run() { + boolean wasInterrupted = false; + while (this.running) { + try { + KafkaMessage message = messages.take(); + try { + delegate.onMessage(message); + } + catch (Exception e) { + if (errorHandler != null) { + errorHandler.handle(e, message); + } + } + finally { + offsetManager.updateOffset(message.getMetadata().getPartition(), + message.getMetadata().getNextOffset()); + } + } + catch (InterruptedException e) { + wasInterrupted = true; + } + } + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + } + +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/package-info.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/package-info.java new file mode 100644 index 0000000000..c2a49222f5 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/package-info.java @@ -0,0 +1,4 @@ +/** + * Provides message listener container support + */ +package org.springframework.integration.kafka.listener; diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaHeaders.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaHeaders.java index 148235812b..eec5496cfe 100644 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaHeaders.java +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaHeaders.java @@ -28,4 +28,8 @@ public abstract class KafkaHeaders { public static final String MESSAGE_KEY = PREFIX + "messageKey"; + public static final String PARTITION_ID = PREFIX + "_partitionId"; + + public static final String OFFSET = PREFIX + "_offset"; + } diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/MessageUtils.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/MessageUtils.java new file mode 100644 index 0000000000..b50db52c19 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/MessageUtils.java @@ -0,0 +1,43 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.util; + +import kafka.serializer.Decoder; +import kafka.utils.Utils$; + +import org.springframework.integration.kafka.core.KafkaMessage; + +/** + * @author Marius Bogoevici + */ +public class MessageUtils { + + public static T decodeKey(KafkaMessage message, Decoder decoder) { + if (message.getMessage().isNull() || !message.getMessage().hasKey()) { + return null; + } + return decoder.fromBytes(Utils$.MODULE$.readBytes(message.getMessage().key())); + } + + public static T decodePayload(KafkaMessage message, Decoder decoder) { + if (message.getMessage().isNull()) { + return null; + } + return decoder.fromBytes(Utils$.MODULE$.readBytes(message.getMessage().payload())); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractBrokerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractBrokerTests.java new file mode 100644 index 0000000000..dcfa537e1f --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractBrokerTests.java @@ -0,0 +1,157 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static scala.collection.JavaConversions.asScalaBuffer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import com.gs.collections.api.RichIterable; +import com.gs.collections.api.block.function.Function2; +import com.gs.collections.api.multimap.Multimap; +import com.gs.collections.api.multimap.MutableMultimap; +import com.gs.collections.api.tuple.Pair; +import com.gs.collections.impl.factory.Multimaps; +import com.gs.collections.impl.tuple.Tuples; +import kafka.admin.AdminUtils; +import kafka.producer.KeyedMessage; +import kafka.producer.Producer; +import kafka.producer.ProducerConfig; +import kafka.serializer.StringEncoder; +import kafka.utils.TestUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.After; +import scala.collection.JavaConversions; +import scala.collection.Map; +import scala.collection.immutable.List$; +import scala.collection.immutable.Map$; +import scala.collection.immutable.Seq; + +import org.springframework.integration.kafka.core.Configuration; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.DefaultConnectionFactory; +import org.springframework.integration.kafka.core.BrokerAddressListConfiguration; +import org.springframework.integration.kafka.rule.KafkaRule; + +/** + * @author Marius Bogoevici + */ +public abstract class AbstractBrokerTests { + + private static final Log log = LogFactory.getLog(AbstractBrokerTests.class); + + public static final String TEST_TOPIC = "test-topic"; + + public abstract KafkaRule getKafkaRule(); + + @After + public void cleanUp() { + deleteTopic(TEST_TOPIC); + } + + @SuppressWarnings("unchecked") + public void createTopic(String topicName, int partitionCount, int brokers, int replication) { + MutableMultimap partitionDistribution = createPartitionDistribution(partitionCount, brokers, replication); + AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK(getKafkaRule().getZkClient(), + topicName, toKafkaPartitionMap(partitionDistribution), + AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK$default$4(), + AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK$default$5()); + if (getKafkaRule().isEmbedded()) { + for (int i = 0; i < partitionDistribution.keysView().size(); i++) { + TestUtils.waitUntilMetadataIsPropagated(asScalaBuffer(getKafkaRule().getKafkaServers()), topicName, i, 5000L); + } + } else { + sleep(partitionCount * 200); + } + } + + public void deleteTopic(String topicName) { + AdminUtils.deleteTopic(getKafkaRule().getZkClient(), topicName); + if (getKafkaRule().isEmbedded()) { + TestUtils.waitUntilMetadataIsPropagated(asScalaBuffer(getKafkaRule().getKafkaServers()), topicName, 0, 5000L); + } else { + sleep(1000); + } + } + + public MutableMultimap createPartitionDistribution(int partitionCount, int brokers, int replication) { + MutableMultimap partitionDistribution = Multimaps.mutable.list.with(); + for (int i = 0; i < partitionCount; i++) { + for (int j = 0; j < replication; j++) { + partitionDistribution.put(i, (i + j) % brokers); + } + } + return partitionDistribution; + } + + + public Configuration getKafkaConfiguration() { + return new BrokerAddressListConfiguration(getKafkaRule().getBrokerAddresses()); + } + + public static scala.collection.Seq> createMessages(int count, String topic) { + return createMessagesInRange(0,count-1,topic); + } + + public static scala.collection.Seq> createMessagesInRange(int start, int end, String topic) { + List> messages = new ArrayList>(); + for (int i=start; i<= end; i++) { + messages.add(new KeyedMessage(topic, "Key " + i, i, "Message " + i)); + } + return asScalaBuffer(messages).toSeq(); + } + + public Producer createStringProducer(int compression) { + Properties producerConfig = TestUtils.getProducerConfig(getKafkaRule().getBrokersAsString(), + TestPartitioner.class.getCanonicalName()); + producerConfig.put("serializer.class", StringEncoder.class.getCanonicalName()); + producerConfig.put("key.serializer.class", StringEncoder.class.getCanonicalName()); + producerConfig.put("compression.codec", Integer.toString(compression)); + return new Producer(new ProducerConfig(producerConfig)); + } + + public ConnectionFactory getKafkaBrokerConnectionFactory() throws Exception { + DefaultConnectionFactory connectionFactory = new DefaultConnectionFactory(getKafkaConfiguration()); + connectionFactory.afterPropertiesSet(); + return connectionFactory; + } + + @SuppressWarnings({"rawtypes", "serial", "deprecation"}) + private Map toKafkaPartitionMap(Multimap partitions) { + java.util.Map> m = partitions.toMap().collect(new Function2, Pair>>() { + @Override + public Pair> value(Integer argument1, RichIterable argument2) { + return Tuples.pair((Object) argument1, List$.MODULE$.fromArray(argument2.toArray(new Object[0])).toSeq()); + } + }); + return Map$.MODULE$.apply(JavaConversions.asScalaMap(m).toSeq()); + } + + private static void sleep(int time) { + try { + Thread.sleep(time); + } + catch (InterruptedException e) { + log.error(e); + } + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractMessageListenerContainerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractMessageListenerContainerTests.java new file mode 100644 index 0000000000..07a9695d0e --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractMessageListenerContainerTests.java @@ -0,0 +1,224 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.collection.IsEmptyCollection.empty; +import static org.hamcrest.core.IsCollectionContaining.hasItem; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; +import static org.springframework.integration.kafka.util.MessageUtils.decodeKey; +import static org.springframework.integration.kafka.util.MessageUtils.decodePayload; + +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.gs.collections.api.RichIterable; +import com.gs.collections.api.bag.MutableBag; +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.block.function.Function2; +import com.gs.collections.api.block.procedure.Procedure; +import com.gs.collections.api.list.MutableList; +import com.gs.collections.api.map.MutableMap; +import com.gs.collections.api.multimap.list.MutableListMultimap; +import com.gs.collections.api.set.MutableSet; +import com.gs.collections.api.tuple.Pair; +import com.gs.collections.impl.factory.Sets; +import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; +import com.gs.collections.impl.tuple.Tuples; +import kafka.serializer.StringDecoder; +import kafka.utils.VerifiableProperties; +import org.hamcrest.Matchers; + +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.KafkaMessage; + +/** + * @author Marius Bogoevici + */ +public abstract class AbstractMessageListenerContainerTests extends AbstractBrokerTests { + + public void runMessageListenerTest(int maxReceiveSize, int concurrency, int partitionCount, int testMessageCount, int divisionFactor, int compressionCodec, String topic) throws Exception { + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + for (int i = 0; i < partitionCount; i++) { + if(i % divisionFactor == 0) { + readPartitions.add(new Partition(topic, i)); + } + } + final KafkaMessageListenerContainer kafkaMessageListenerContainer = new KafkaMessageListenerContainer(connectionFactory, readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(maxReceiveSize); + kafkaMessageListenerContainer.setConcurrency(concurrency); + + int expectedMessageCount = testMessageCount / divisionFactor; + + final MutableListMultimap receivedData = new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + kafkaMessageListenerContainer.setMessageListener(new MessageListener() { + @Override + public void onMessage(KafkaMessage message) { + StringDecoder decoder = new StringDecoder(new VerifiableProperties()); + receivedData.put(message.getMetadata().getPartition().getId(),new KeyedMessageWithOffset(decodeKey(message, decoder), decodePayload(message, decoder), message.getMetadata().getOffset(), Thread.currentThread().getName(), message.getMetadata().getPartition().getId())); + latch.countDown(); + } + }); + + kafkaMessageListenerContainer.start(); + + createStringProducer(compressionCodec).send(createMessages(testMessageCount, topic)); + + latch.await((expectedMessageCount/5000) + 1, TimeUnit.MINUTES); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, concurrency, partitionCount, testMessageCount, expectedMessageCount, readPartitions, divisionFactor); + + } + + @SuppressWarnings("serial") + public void validateMessageReceipt(MutableListMultimap receivedData, int concurrency, int partitionCount, int testMessageCount, int expectedMessageCount, ArrayList readPartitions, int divisionFactor) { + // Group messages received by processing thread + MutableListMultimap messagesByThread = receivedData.valuesView().toList().groupBy(new Function() { + @Override + public String valueOf(KeyedMessageWithOffset object) { + return object.getThreadName(); + } + }); + + // Execution has taken place on as many distinct threads as configured + assertThat(messagesByThread.keysView().size(), Matchers.equalTo(concurrency)); + + // Group partitions by thread + MutableMap> partitionsByThread = messagesByThread.toMap().collect(new Function2, Pair>>() { + @Override + public Pair> value(String argument1, RichIterable argument2) { + return Tuples.pair(argument1, argument2.collect(new Function() { + @Override + public Integer valueOf(KeyedMessageWithOffset object) { + return object.getPartition(); + } + }).toSet()); + } + }); + + // Messages from a partition have been executed on the same thread and groups are mutually exclusive + final MutableSet validatedPartitions = Sets.mutable.of(); + partitionsByThread.valuesView().forEach(new Procedure>() { + @Override + public void value(MutableSet partitions) { + assertThat(validatedPartitions.intersect(partitions), empty()); + validatedPartitions.addAll(partitions); + } + }); + + // All partitions are accounted for, but only the ones that we were expecting to read from + for (int i = 0; i < partitionCount; i++) { + if (i % divisionFactor == 0) { + assertThat(validatedPartitions, hasItem(i)); + } else { + assertThat(validatedPartitions, not(hasItem(i))); + } + } + + // Sort data by payload in order to identify duplicates + MutableList sortedPayloads = receivedData.valuesView().toList().collect(new Function() { + @Override + public String valueOf(KeyedMessageWithOffset object) { + return object.getPayload(); + } + }).sortThis(); + + // Remove unique values - what is left are duplicates + MutableBag duplicates = sortedPayloads.toBag(); + duplicates.removeAll(sortedPayloads.toSet()); + + // The final set has exactly the same size as the message count + assertThat(sortedPayloads, hasSize(expectedMessageCount)); + + // There are no duplicates - all messages have been received only once + assertThat(duplicates, hasSize(0)); + + // Group offsets by partition + MutableMap> offsetsByPartition = receivedData.toMap().collect(new Function2, Pair>>() { + @Override + public Pair> value(Integer partition, RichIterable argument2) { + return Tuples.pair(partition, argument2.collect(new Function() { + @Override + public Long valueOf(KeyedMessageWithOffset object) { + return object.getOffset(); + } + }).toList()); + } + }); + + // Check that the sequence of offsets has been processed in ascending order, with no gaps + for (MutableList offsetsForPartition : offsetsByPartition.valuesView()) { + for (int i = 0; i < offsetsForPartition.size() - 1; i++) { + assertThat(offsetsForPartition.get(i+1), equalTo(offsetsForPartition.get(i) + 1)); + } + } + } + + static class KeyedMessageWithOffset { + + String key; + + String payload; + + Long offset; + + String threadName; + + int partition; + + public KeyedMessageWithOffset(String key, String payload, Long offset, String threadName, int partition) { + this.key = key; + this.payload = payload; + this.offset = offset; + this.threadName = threadName; + this.partition = partition; + } + + public String getKey() { + return key; + } + + public String getPayload() { + return payload; + } + + public Long getOffset() { + return offset; + } + + public String getThreadName() { + return threadName; + } + + public int getPartition() { + return partition; + } + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests-context.xml b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests-context.xml new file mode 100644 index 0000000000..d1ddd66a41 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests-context.xml @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests.java new file mode 100644 index 0000000000..709436c5fb --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests.java @@ -0,0 +1,71 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.Matchers.notNullValue; + + +import kafka.message.NoCompressionCodec$; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.context.support.ClassPathXmlApplicationContext; +import org.springframework.integration.channel.QueueChannel; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.rule.KafkaRule; +import org.springframework.messaging.Message; + +/** + * @author Marius Bogoevici + */ +public class ChannelAdapterWithXmlConfigurationTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaRule kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaRule getKafkaRule() { + return this.kafkaEmbeddedBrokerRule; + } + + @Test + public void testConsumptionWithXmlConfiguration() throws Exception { + + System.setProperty("kafka.test.port", + String.valueOf(kafkaEmbeddedBrokerRule.getBrokerAddresses().get(0).getPort())); + System.setProperty("kafka.test.topic", TEST_TOPIC); + + createTopic(TEST_TOPIC, 1, 1, 1); + + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessages(100, TEST_TOPIC)); + + ClassPathXmlApplicationContext context = + new ClassPathXmlApplicationContext("ChannelAdapterWithXmlConfigurationTests-context.xml", + ChannelAdapterWithXmlConfigurationTests.class); + + QueueChannel output = context.getBean("output", QueueChannel.class); + + for (int i = 0; i < 100; i++) { + Message received = output.receive(1000); + Assert.assertThat(received, notNullValue()); + } + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelSendingMessageListener.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelSendingMessageListener.java new file mode 100644 index 0000000000..9eda772104 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelSendingMessageListener.java @@ -0,0 +1,61 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.listener.MessageListener; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.integration.support.channel.BeanFactoryChannelResolver; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.core.DestinationResolver; + +/** + * @author Marius Bogoevici + */ +public class ChannelSendingMessageListener implements MessageListener, ApplicationContextAware { + + private String channelName; + + private MessageChannel messageChannel; + + public String getChannelName() { + return channelName; + } + + public void setChannelName(String channelName) { + this.channelName = channelName; + } + + DestinationResolver destinationResolver; + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.destinationResolver = new BeanFactoryChannelResolver(applicationContext); + messageChannel = destinationResolver.resolveDestination(channelName); + } + + @Override + public void onMessage(KafkaMessage message) { + byte b[] = new byte[message.getMessage().payloadSize()]; + message.getMessage().payload().get(b); + messageChannel.send(MessageBuilder.withPayload(new String(b)).build()); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionFactoryTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionFactoryTests.java new file mode 100644 index 0000000000..5ec2b7a2be --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionFactoryTests.java @@ -0,0 +1,87 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.collection.IsEmptyCollection.empty; +import static org.junit.Assert.assertThat; + +import java.util.List; + +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.core.BrokerAddress; +import org.springframework.integration.kafka.core.Connection; +import org.springframework.integration.kafka.core.DefaultConnectionFactory; +import org.springframework.integration.kafka.core.BrokerAddressListConfiguration; +import org.springframework.integration.kafka.core.Result; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.ZookeeperConfiguration; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.support.ZookeeperConnect; + +/** + * @author Marius Bogoevici + */ +public class DefaultConnectionFactoryTests extends AbstractBrokerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testCreateConnectionFactoryWithBrokerList() throws Exception { + + createTopic(TEST_TOPIC, 1, 1, 1); + + List brokerAddresses = getKafkaRule().getBrokerAddresses(); + Partition partition = new Partition(TEST_TOPIC, 0); + DefaultConnectionFactory connectionFactory = new DefaultConnectionFactory(new BrokerAddressListConfiguration(brokerAddresses)); + connectionFactory.afterPropertiesSet(); + Connection connection = connectionFactory.connect(getKafkaRule().getBrokerAddresses().get(0)); + Result leaders = connection.findLeaders(TEST_TOPIC); + assertThat(leaders.getErrors().entrySet(), empty()); + assertThat(leaders.getResults().entrySet(), hasSize(1)); + assertThat(leaders.getResults().get(partition), equalTo(getKafkaRule().getBrokerAddresses().get(0))); + } + + @Test + public void testCreateConnectionFactoryWithZookeeper() throws Exception { + + createTopic(TEST_TOPIC, 1, 1, 1); + + + Partition partition = new Partition(TEST_TOPIC, 0); + ZookeeperConnect zookeeperConnect = new ZookeeperConnect(); + zookeeperConnect.setZkConnect(kafkaEmbeddedBrokerRule.getZookeeperConnectionString()); + DefaultConnectionFactory connectionFactory = + new DefaultConnectionFactory(new ZookeeperConfiguration(zookeeperConnect)); + connectionFactory.afterPropertiesSet(); + Connection connection = connectionFactory.connect(getKafkaRule().getBrokerAddresses().get(0)); + Result leaders = connection.findLeaders(TEST_TOPIC); + assertThat(leaders.getErrors().entrySet(), empty()); + assertThat(leaders.getResults().entrySet(), hasSize(1)); + assertThat(leaders.getResults().get(partition), equalTo(getKafkaRule().getBrokerAddresses().get(0))); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionTests.java new file mode 100644 index 0000000000..51e06fcbe1 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionTests.java @@ -0,0 +1,126 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import kafka.producer.Producer; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.core.Connection; +import org.springframework.integration.kafka.core.DefaultConnection; +import org.springframework.integration.kafka.core.FetchRequest; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.KafkaMessageBatch; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.Result; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.serializer.common.StringDecoder; +import org.springframework.integration.kafka.util.MessageUtils; + +/** + * @author Marius Bogoevici + */ +public class DefaultConnectionTests extends AbstractBrokerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testFetchPartitionMetadata() throws Exception { + createTopic(TEST_TOPIC, 1, 1, 1); + Connection brokerConnection = new DefaultConnection(getKafkaRule().getBrokerAddresses().get(0), "client", 64*1024, 3000, 1, 100); + Partition partition = new Partition(TEST_TOPIC, 0); + Result result = brokerConnection.fetchInitialOffset(-1, partition); + Assert.assertEquals(0, result.getErrors().size()); + Assert.assertEquals(1, result.getResults().size()); + Assert.assertEquals(Long.valueOf(0), result.getResults().get(partition)); + } + + @Test + public void testReceiveMessages() throws Exception { + createTopic(TEST_TOPIC, 1, 1, 1); + Producer producer = createStringProducer(0); + producer.send( createMessages(10, TEST_TOPIC)); + Connection brokerConnection = new DefaultConnection(getKafkaRule().getBrokerAddresses().get(0), "client", 64*1024, 3000, 1, 100); + Partition partition = new Partition(TEST_TOPIC, 0); + FetchRequest fetchRequest = new FetchRequest(partition, 0L, 1000); + Result result = brokerConnection.fetch(fetchRequest); + Assert.assertEquals(0, result.getErrors().size()); + Assert.assertEquals(1, result.getResults().size()); + Assert.assertEquals(10, result.getResults().get(partition).getMessages().size()); + Assert.assertEquals(10,result.getResults().get(partition).getHighWatermark()); + StringDecoder decoder = new StringDecoder(); + int i = 0; + for (KafkaMessage kafkaMessage : result.getResults().get(partition).getMessages()) { + Assert.assertEquals("Key " + i, MessageUtils.decodeKey(kafkaMessage, decoder)); + Assert.assertEquals("Message " + i, MessageUtils.decodePayload(kafkaMessage, decoder)); + i++; + } + } + + @Test + public void testReceiveMessagesWithCompression1() throws Exception { + createTopic(TEST_TOPIC, 1, 1, 1); + Producer producer = createStringProducer(1); + producer.send( createMessages(10, TEST_TOPIC)); + Connection brokerConnection = new DefaultConnection(getKafkaRule().getBrokerAddresses().get(0), "client", 64*1024, 3000, 1, 100); + Partition partition = new Partition(TEST_TOPIC, 0); + FetchRequest fetchRequest = new FetchRequest(partition, 0L, 1000); + Result result = brokerConnection.fetch(fetchRequest); + Assert.assertEquals(0, result.getErrors().size()); + Assert.assertEquals(1, result.getResults().size()); + Assert.assertEquals(10, result.getResults().get(partition).getMessages().size()); + Assert.assertEquals(10,result.getResults().get(partition).getHighWatermark()); + StringDecoder decoder = new StringDecoder(); + int i = 0; + for (KafkaMessage kafkaMessage : result.getResults().get(partition).getMessages()) { + Assert.assertEquals("Key " + i, MessageUtils.decodeKey(kafkaMessage, decoder)); + Assert.assertEquals("Message " + i, MessageUtils.decodePayload(kafkaMessage, decoder)); + i++; + } + } + + @Test + public void testReceiveMessagesWithCompression2() throws Exception { + createTopic(TEST_TOPIC, 1, 1, 1); + Producer producer = createStringProducer(2); + producer.send( createMessages(10, TEST_TOPIC)); + Connection brokerConnection = new DefaultConnection(getKafkaRule().getBrokerAddresses().get(0), "client", 64*1024, 3000, 1, 100); + Partition partition = new Partition(TEST_TOPIC, 0); + FetchRequest fetchRequest = new FetchRequest(partition, 0L, 1000); + Result result = brokerConnection.fetch(fetchRequest); + Assert.assertEquals(0, result.getErrors().size()); + Assert.assertEquals(1, result.getResults().size()); + Assert.assertEquals(10, result.getResults().get(partition).getMessages().size()); + Assert.assertEquals(10,result.getResults().get(partition).getHighWatermark()); + StringDecoder decoder = new StringDecoder(); + int i = 0; + for (KafkaMessage kafkaMessage : result.getResults().get(partition).getMessages()) { + Assert.assertEquals("Key " + i, MessageUtils.decodeKey(kafkaMessage, decoder)); + Assert.assertEquals("Message " + i, MessageUtils.decodePayload(kafkaMessage, decoder)); + i++; + } + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterTests.java new file mode 100644 index 0000000000..3760444eb8 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterTests.java @@ -0,0 +1,126 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; + +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.gs.collections.api.multimap.list.MutableListMultimap; +import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; +import kafka.message.NoCompressionCodec$; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.beans.factory.BeanFactory; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.rule.KafkaRule; +import org.springframework.integration.kafka.serializer.common.StringDecoder; +import org.springframework.integration.kafka.support.KafkaHeaders; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; + +/** + * @author Marius Bogoevici + */ +public class KafkaMessageDrivenChannelAdapterTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaRule getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + @SuppressWarnings("serial") + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + for (int i = 0; i < 5; i++) { + readPartitions.add(new Partition(TEST_TOPIC, i)); + } + + final KafkaMessageListenerContainer kafkaMessageListenerContainer = + new KafkaMessageListenerContainer(connectionFactory, + readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + + int expectedMessageCount = 100; + + final MutableListMultimap receivedData = + new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + + KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = + new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); + + StringDecoder decoder = new StringDecoder(); + kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); + kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { + @Override + public boolean send(Message message) { + latch.countDown(); + return receivedData.put( + (Integer)message.getHeaders().get(KafkaHeaders.PARTITION_ID), + new KeyedMessageWithOffset( + (String)message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), + (String)message.getPayload(), + (Long)message.getHeaders().get(KafkaHeaders.OFFSET), + Thread.currentThread().getName(), + (Integer)message.getHeaders().get(KafkaHeaders.PARTITION_ID))); + } + + + @Override + public boolean send(Message message, long timeout) { + return send(message); + } + }); + + kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); + kafkaMessageDrivenChannelAdapter.start(); + + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessages(100, TEST_TOPIC)); + + latch.await((expectedMessageCount/5000) + 1, TimeUnit.MINUTES); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, 2, 5, 100, expectedMessageCount, readPartitions, 1); + + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests.java new file mode 100644 index 0000000000..e8665f9073 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests.java @@ -0,0 +1,172 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.gs.collections.api.RichIterable; +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.multimap.list.MutableListMultimap; +import com.gs.collections.api.tuple.Pair; +import com.gs.collections.impl.list.mutable.FastList; +import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; +import com.gs.collections.impl.utility.Iterate; + +import kafka.message.NoCompressionCodec$; + +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.beans.factory.BeanFactory; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.rule.KafkaRule; +import org.springframework.integration.kafka.serializer.common.StringDecoder; +import org.springframework.integration.kafka.support.KafkaHeaders; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; + +/** + * @author Marius Bogoevici + */ +public class KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaRule getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + @SuppressWarnings("serial") + public void testLowVolumeLowConcurrency() throws Exception { + + // we will send 300 messages: first 200, then another 100 + // we will start reading from all partitions at offset 100 + int expectedMessageCount = 200; + + createTopic(TEST_TOPIC, 5, 1, 1); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + Map startingOffsets = new HashMap(); + for (int i = 0; i < 5; i++) { + Partition partition = new Partition(TEST_TOPIC, i); + readPartitions.add(partition); + startingOffsets.put(partition, 20L); + } + + final KafkaMessageListenerContainer kafkaMessageListenerContainer = + new KafkaMessageListenerContainer(connectionFactory, + readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + MetadataStoreOffsetManager offsetManager = new MetadataStoreOffsetManager(connectionFactory, startingOffsets); + kafkaMessageListenerContainer.setOffsetManager(offsetManager); + + // we send 100 messages + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessagesInRange(0, 199, TEST_TOPIC)); + + final MutableListMultimap receivedData = + new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + + KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = + new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); + + StringDecoder decoder = new StringDecoder(); + kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); + kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { + @Override + public boolean send(Message message) { + latch.countDown(); + return receivedData.put( + (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID), + new KeyedMessageWithOffset( + (String) message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), + (String) message.getPayload(), + (Long) message.getHeaders().get(KafkaHeaders.OFFSET), + Thread.currentThread().getName(), + (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID))); + } + + + @Override + public boolean send(Message message, long timeout) { + return send(message); + } + }); + + kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); + kafkaMessageDrivenChannelAdapter.start(); + + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessagesInRange(200, 299, TEST_TOPIC)); + + latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, 2, 5, 100, expectedMessageCount, readPartitions, 1); + + // For all received messages + Collection allReceivedMessages = + Iterate.flatCollect(receivedData.keyMultiValuePairsView(), + new Function>, + RichIterable>() { + + @Override + public RichIterable valueOf(Pair> object) { + return object.getTwo(); + } + + }); + + // We extract the sequence value, i.e. "Message xx" + Integer minValueInMessage = FastList.newList(allReceivedMessages).collect(new Function() { + @Override + public Integer valueOf(KeyedMessageWithOffset object) { + return Integer.parseInt(object.getPayload().split(" ")[1]); + } + }).min(); + + // The lowest received value is 100. That is correct, because messages are evenly distributed across partitions + // and we start reading only at partition 200 + assertThat(minValueInMessage, equalTo(100)); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithWrongOffsetTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithWrongOffsetTests.java new file mode 100644 index 0000000000..ad85926afd --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithWrongOffsetTests.java @@ -0,0 +1,319 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.gs.collections.api.RichIterable; +import com.gs.collections.api.block.function.Function; +import com.gs.collections.api.multimap.list.MutableListMultimap; +import com.gs.collections.api.tuple.Pair; +import com.gs.collections.impl.list.mutable.FastList; +import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; +import com.gs.collections.impl.utility.Iterate; + +import kafka.message.NoCompressionCodec$; + +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.beans.factory.BeanFactory; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.rule.KafkaRule; +import org.springframework.integration.kafka.serializer.common.StringDecoder; +import org.springframework.integration.kafka.support.KafkaHeaders; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.util.StringUtils; + +/** + * @author Marius Bogoevici + */ +public class KafkaMessageDrivenChannelAdapterWithWrongOffsetTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaRule getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + @SuppressWarnings("serial") + public void testLowVolumeLowConcurrencyResetAtEarliest() throws Exception { + + // we will send 300 messages: first 200, then another 100 + // we will start reading from all partitions at offset 100 + int expectedMessageCount = 200; + + createTopic(TEST_TOPIC, 5, 1, 1); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + Map startingOffsets = new HashMap(); + for (int i = 0; i < 5; i++) { + Partition partition = new Partition(TEST_TOPIC, i); + readPartitions.add(partition); + startingOffsets.put(partition, 900L); + } + + final KafkaMessageListenerContainer kafkaMessageListenerContainer = + new KafkaMessageListenerContainer(connectionFactory, + readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + MetadataStoreOffsetManager offsetManager = new MetadataStoreOffsetManager(connectionFactory, startingOffsets); + kafkaMessageListenerContainer.setOffsetManager(offsetManager); + + // we send 100 messages + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessagesInRange(0, 199, TEST_TOPIC)); + + final MutableListMultimap receivedData = + new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + + KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = + new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); + + StringDecoder decoder = new StringDecoder(); + kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); + + kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { + @Override + public boolean send(Message message) { + latch.countDown(); + return receivedData.put( + (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID), + new KeyedMessageWithOffset( + (String) message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), + (String) message.getPayload(), + (Long) message.getHeaders().get(KafkaHeaders.OFFSET), + Thread.currentThread().getName(), + (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID))); + } + + + @Override + public boolean send(Message message, long timeout) { + return send(message); + } + }); + + kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); + kafkaMessageDrivenChannelAdapter.start(); + + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessagesInRange(200, 299, TEST_TOPIC)); + + latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, 2, 5, 100, expectedMessageCount, readPartitions, 1); + + // For all received messages + Collection allReceivedMessages = + Iterate.flatCollect(receivedData.keyMultiValuePairsView(), + new Function>, RichIterable>() { + + @Override + public RichIterable valueOf(Pair> object) { + return object.getTwo(); + } + + }); + + // We extract the sequence value, i.e. "Message xx" + Integer minValueInMessage = FastList.newList(allReceivedMessages) + .collect(new Function() { + @Override + public Integer valueOf(KeyedMessageWithOffset object) { + return Integer.parseInt(object.getPayload().split(" ")[1]); + } + }).min(); + + // The lowest received value is 0. That is correct, because reading started at 0 after the reset + assertThat(minValueInMessage, equalTo(0)); + } + + @Test + @SuppressWarnings("serial") + public void testLowVolumeLowConcurrencyResetAtLatest() throws Exception { + + // we will send 300 messages: first 200, then another 100 + // we will start reading from all partitions at offset 900, but since that is reset + int expectedMessageCount = 100; + + createTopic(TEST_TOPIC, 5, 1, 1); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + Map startingOffsets = new HashMap(); + for (int i = 0; i < 5; i++) { + Partition partition = new Partition(TEST_TOPIC, i); + readPartitions.add(partition); + startingOffsets.put(partition, 900L); + } + + final KafkaMessageListenerContainer kafkaMessageListenerContainer = + new KafkaMessageListenerContainer(connectionFactory, + readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + final MetadataStoreOffsetManager offsetManager = + new MetadataStoreOffsetManager(connectionFactory, startingOffsets); + offsetManager.setReferenceTimestamp(-1); + final CountDownLatch offsetResetLatch = new CountDownLatch(1); + + // Manually generated proxy for catching the reset event. We only want to send data once we are + // sure that the OffsetManager has been reset. Otherwise the test may not finish (due to pollers starting after send + + OffsetManager offsetManagerWrapper = new OffsetManager() { + + @Override + public void flush() throws IOException { + offsetManager.flush(); + } + + @Override + public void close() throws IOException { + offsetManager.close(); + } + + @Override + public void resetOffsets(Collection partitionsToReset) { + System.out.println("resetting " + offsetResetLatch.getCount() + " partitions: " + + StringUtils.collectionToDelimitedString(partitionsToReset, ",")); + offsetManager.resetOffsets(partitionsToReset); + offsetResetLatch.countDown(); + } + + @Override + public long getOffset(Partition partition) { + long offset = offsetManager.getOffset(partition); + System.out.println("Result for " + partition + " offset: " + offset); + return offset; + } + + @Override + public void updateOffset(Partition partition, long offset) { + offsetManager.updateOffset(partition, offset); + } + + }; + + kafkaMessageListenerContainer.setOffsetManager(offsetManagerWrapper); + + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessagesInRange(0, 199, TEST_TOPIC)); + + final MutableListMultimap receivedData = + new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + + KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = + new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); + + StringDecoder decoder = new StringDecoder(); + kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); + kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { + @Override + public boolean send(Message message) { + latch.countDown(); + return receivedData.put( + (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID), + new KeyedMessageWithOffset( + (String) message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), + (String) message.getPayload(), + (Long) message.getHeaders().get(KafkaHeaders.OFFSET), + Thread.currentThread().getName(), + (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID))); + } + + @Override + public boolean send(Message message, long timeout) { + return send(message); + } + }); + + kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); + kafkaMessageDrivenChannelAdapter.start(); + + // we wait for the reset event first + offsetResetLatch.await(1000, TimeUnit.MILLISECONDS); + Thread.sleep(1000); + createStringProducer(NoCompressionCodec$.MODULE$.codec()).send(createMessagesInRange(200, 299, TEST_TOPIC)); + + latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, 2, 5, 100, expectedMessageCount, readPartitions, 1); + + // For all received messages + Collection allReceivedMessages = + Iterate.flatCollect(receivedData.keyMultiValuePairsView(), + new Function>, RichIterable>() { + + @Override + public RichIterable valueOf(Pair> object) { + return object.getTwo(); + } + + }); + + // We extract the sequence value, i.e. "Message xx" + Integer minValueInMessage = FastList.newList(allReceivedMessages) + .collect(new Function() { + + @Override + public Integer valueOf(KeyedMessageWithOffset object) { + return Integer.parseInt(object.getPayload().split(" ")[1]); + } + + }) + .min(); + + // The lowest received value is 0. That is correct, because reading started at 0 after the reset + assertThat(minValueInMessage, equalTo(200)); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker20Tests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker20Tests.java new file mode 100644 index 0000000000..2ef274c065 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker20Tests.java @@ -0,0 +1,60 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ +public class MultiBroker20Tests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(20); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 20, 1); + runMessageListenerTest(100, 20, 100, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 20, 1); + runMessageListenerTest(100, 20, 100, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 20, 1); + runMessageListenerTest(100, 20, 100, 100000, 1, 0, TEST_TOPIC); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker2Tests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker2Tests.java new file mode 100644 index 0000000000..5c16d3a3f1 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker2Tests.java @@ -0,0 +1,103 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.rule.KafkaRule; + +/** + * @author Marius Bogoevici + */ +public class MultiBroker2Tests extends AbstractMessageListenerContainerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(2); + + @Override + public KafkaRule getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + runMessageListenerTest(100, 2, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + runMessageListenerTest(100, 2, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + runMessageListenerTest(100, 2, 5, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testLowVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + runMessageListenerTest(100, 5, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + runMessageListenerTest(100, 5, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + runMessageListenerTest(100, 5, 5, 100000, 1, 0, TEST_TOPIC); + } + + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 2, 1); + runMessageListenerTest(100, 20, 100, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 2, 1); + runMessageListenerTest(100, 20, 100, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 2, 1); + runMessageListenerTest(100, 20, 100, 100000, 1, 0, TEST_TOPIC); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedTests.java new file mode 100644 index 0000000000..b3023bf515 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedTests.java @@ -0,0 +1,102 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ +public class MultiBroker5ReplicatedTests extends AbstractMessageListenerContainerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(5); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 3); + runMessageListenerTest(100, 2, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 3); + runMessageListenerTest(100, 2, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 3); + runMessageListenerTest(100, 2, 5, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testLowVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 3); + runMessageListenerTest(100, 5, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 3); + runMessageListenerTest(100, 5, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 5, 5, 100000, 1, 0, TEST_TOPIC); + } + + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 5, 3); + runMessageListenerTest(100, 20, 100, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 5, 3); + runMessageListenerTest(100, 20, 100, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 5, 3); + runMessageListenerTest(100, 20, 100, 100000, 1, 0, TEST_TOPIC); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedWithBounceTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedWithBounceTests.java new file mode 100644 index 0000000000..6451179d71 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedWithBounceTests.java @@ -0,0 +1,114 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; +import static org.springframework.integration.kafka.util.MessageUtils.decodeKey; +import static org.springframework.integration.kafka.util.MessageUtils.decodePayload; + +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.gs.collections.api.multimap.list.MutableListMultimap; +import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; + +import kafka.serializer.StringDecoder; +import kafka.utils.VerifiableProperties; + +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ + +public class MultiBroker5ReplicatedWithBounceTests extends AbstractMessageListenerContainerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(5); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 3); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + for (int i = 0; i < 5; i++) { + readPartitions.add(new Partition(TEST_TOPIC, i)); + } + final KafkaMessageListenerContainer kafkaMessageListenerContainer = + new KafkaMessageListenerContainer(connectionFactory, + readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + + final int expectedMessageCount = 100; + + createStringProducer(0).send(createMessages(100, TEST_TOPIC)); + + + final MutableListMultimap receivedData = + new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + kafkaMessageListenerContainer.setMessageListener(new MessageListener() { + + @Override + public void onMessage(KafkaMessage message) { + StringDecoder decoder = new StringDecoder(new VerifiableProperties()); + receivedData.put(message.getMetadata().getPartition().getId(), + new KeyedMessageWithOffset(decodeKey(message, decoder), decodePayload(message, decoder), + message.getMetadata().getOffset(), Thread.currentThread().getName(), + message.getMetadata().getPartition().getId())); + latch.countDown(); + } + + }); + + kafkaMessageListenerContainer.start(); + + for (int i = 0; i < 5; i++) { + if (i % 2 == 0) { + kafkaEmbeddedBrokerRule.bounce(i); + } + } + + latch.await(50, TimeUnit.SECONDS); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, 2, 5, 100, expectedMessageCount, readPartitions, 1); + + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5Tests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5Tests.java new file mode 100644 index 0000000000..8bfc351fb6 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5Tests.java @@ -0,0 +1,102 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ +public class MultiBroker5Tests extends AbstractMessageListenerContainerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(5); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 2, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 2, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 2, 5, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testLowVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 5, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 5, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 5, 1); + runMessageListenerTest(100, 5, 5, 100000, 1, 0, TEST_TOPIC); + } + + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 5, 1); + runMessageListenerTest(100, 20, 100, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 5, 1); + runMessageListenerTest(100, 20, 100, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 5, 1); + runMessageListenerTest(100, 20, 100, 100000, 1, 0, TEST_TOPIC); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerExternalTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerExternalTests.java new file mode 100644 index 0000000000..32520f9961 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerExternalTests.java @@ -0,0 +1,131 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import java.util.UUID; + +import org.junit.After; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaRule; +import org.springframework.integration.kafka.rule.KafkaRunning; + +/** + * @author Marius Bogoevici + */ +@Ignore +public class SingleBrokerExternalTests extends AbstractMessageListenerContainerTests { + + @ClassRule + public static final KafkaRunning kafkaRunningRule = KafkaRunning.isRunning(); + + @Override + public KafkaRule getKafkaRule() { + return kafkaRunningRule; + } + + @Override + @After + public void cleanUp() { + // do nothing, each topic must be created individually + } + + private String generateTopicName() { + return "test" + UUID.randomUUID().toString().replace("-",""); + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 5, 1, 1); + runMessageListenerTest(100, 2, 5, 100, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + public void testMediumVolumeLowConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 5, 1, 1); + runMessageListenerTest(100, 2, 5, 1000, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 5, 1, 1); + runMessageListenerTest(100, 2, 5, 10000, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + public void testLowVolumeMediumConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 5, 1, 1); + runMessageListenerTest(100, 5, 5, 100, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + public void testMediumVolumeMediumConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 5, 1, 1); + runMessageListenerTest(100, 5, 5, 1000, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 5, 1, 1); + runMessageListenerTest(100, 5, 5, 100000, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 1000, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 10000, 1, 0, topicName); + deleteTopic(topicName); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + String topicName = generateTopicName(); + createTopic(topicName, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 100000, 1, 0, topicName); + deleteTopic(topicName); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerTests.java new file mode 100644 index 0000000000..963d453a70 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerTests.java @@ -0,0 +1,98 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ +public class SingleBrokerTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(100, 2, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + public void testMediumVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(100, 2, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(100, 2, 5, 10000, 1, 0, TEST_TOPIC); + } + + @Test + public void testLowVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(100, 5, 5, 100, 1, 0, TEST_TOPIC); + } + + @Test + public void testMediumVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(100, 5, 5, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(100, 5, 5, 100000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 1000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 10000, 1, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 100000, 1, 0, TEST_TOPIC); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithCompressionTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithCompressionTests.java new file mode 100644 index 0000000000..b4ae51c62a --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithCompressionTests.java @@ -0,0 +1,98 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ +public class SingleBrokerWithCompressionTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(10000, 2, 5, 100, 1, 1, TEST_TOPIC); + } + + @Test + public void testMediumVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(10000, 2, 5, 1000, 1, 1, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(10000, 2, 5, 10000, 1, 1, TEST_TOPIC); + } + + @Test + public void testLowVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(10000, 5, 5, 100, 1, 1, TEST_TOPIC); + } + + @Test + public void testMediumVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(10000, 5, 5, 1000, 1, 1, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + runMessageListenerTest(10000, 5, 5, 100000, 1, 1, TEST_TOPIC); + } + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(10000, 20, 100, 1000, 1, 1, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(10000, 20, 100, 10000, 1, 1, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(10000, 20, 100, 100000, 1, 1, TEST_TOPIC); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithPartitionSubsetTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithPartitionSubsetTests.java new file mode 100644 index 0000000000..7f0235122d --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithPartitionSubsetTests.java @@ -0,0 +1,103 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.rule.KafkaEmbedded; + +/** + * @author Marius Bogoevici + */ +public class SingleBrokerWithPartitionSubsetTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 4, 1, 1); + runMessageListenerTest(100, 2, 4, 100, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 4, 1, 1); + runMessageListenerTest(100, 2, 4, 1000, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 4, 1, 1); + runMessageListenerTest(100, 2, 4, 10000, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testLowVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 10, 1, 1); + runMessageListenerTest(100, 5, 10, 100, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 10, 1, 1); + runMessageListenerTest(100, 5, 10, 1000, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeMediumConcurrency() throws Exception { + createTopic(TEST_TOPIC, 10, 1, 1); + runMessageListenerTest(100, 5, 10, 100000, 2, 0, TEST_TOPIC); + } + + + @Test + @Ignore + public void testLowVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 1000, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testMediumVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 10000, 2, 0, TEST_TOPIC); + } + + @Test + @Ignore + public void testHighVolumeHighConcurrency() throws Exception { + createTopic(TEST_TOPIC, 100, 1, 1); + runMessageListenerTest(100, 20, 100, 100000, 2, 0, TEST_TOPIC); + } + + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/TestPartitioner.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/TestPartitioner.java new file mode 100644 index 0000000000..46ff1f5e71 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/TestPartitioner.java @@ -0,0 +1,50 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import kafka.producer.Partitioner; +import kafka.utils.VerifiableProperties; + +/** + * @author Marius Bogoevici + */ +public class TestPartitioner implements Partitioner { + + public TestPartitioner(VerifiableProperties properties) { + } + + @Override + public int partition(Object key, int numPartitions) { + if (key != null) { + if (key instanceof Number) { + return ((Number) key).intValue() % numPartitions; + } + else { + try { + return Integer.parseInt(key.toString()); + } + catch (NumberFormatException e) { + return 0; + } + } + } + else { + return 0; + } + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/WrongTopicNameTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/WrongTopicNameTests.java new file mode 100644 index 0000000000..b0ca1e59a0 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/WrongTopicNameTests.java @@ -0,0 +1,90 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.beans.factory.BeanFactory; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.TopicNotFoundException; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.rule.KafkaRule; +import org.springframework.integration.kafka.serializer.common.StringDecoder; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; + +/** + * @author Marius Bogoevici + */ +public class WrongTopicNameTests extends AbstractMessageListenerContainerTests { + + @Rule + public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); + + @Override + public KafkaRule getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Test(expected = TopicNotFoundException.class) + @SuppressWarnings("serial") + public void testWrongTopicNameFails() throws Exception { + createTopic(TEST_TOPIC, 5, 1, 1); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + + final KafkaMessageListenerContainer kafkaMessageListenerContainer + = new KafkaMessageListenerContainer(connectionFactory, "WRONG-TOPIC"); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + + KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = + new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); + + StringDecoder decoder = new StringDecoder(); + kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); + kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); + kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { + @Override + public boolean send(Message message) { + fail(); + return true; + } + + @Override + public boolean send(Message message, long timeout) { + fail(); + return true; + } + }); + + kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); + kafkaMessageDrivenChannelAdapter.start(); + + + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ZookeeperConfigurationTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ZookeeperConfigurationTests.java new file mode 100644 index 0000000000..3e00ec9311 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ZookeeperConfigurationTests.java @@ -0,0 +1,107 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.listener; + +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; +import static org.springframework.integration.kafka.util.MessageUtils.decodeKey; +import static org.springframework.integration.kafka.util.MessageUtils.decodePayload; + +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.gs.collections.api.multimap.list.MutableListMultimap; +import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; +import kafka.serializer.StringDecoder; +import kafka.utils.VerifiableProperties; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.kafka.core.Configuration; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.ZookeeperConfiguration; +import org.springframework.integration.kafka.rule.KafkaEmbedded; +import org.springframework.integration.kafka.support.ZookeeperConnect; + +/** + * @author Marius Bogoevici + */ +public class ZookeeperConfigurationTests extends AbstractMessageListenerContainerTests { + + @Rule + public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(2); + + @Override + public KafkaEmbedded getKafkaRule() { + return kafkaEmbeddedBrokerRule; + } + + @Override + public Configuration getKafkaConfiguration() { + ZookeeperConnect zookeeperConnect = new ZookeeperConnect(); + zookeeperConnect.setZkConnect(kafkaEmbeddedBrokerRule.getZookeeperConnectionString()); + return new ZookeeperConfiguration(zookeeperConnect); + } + + @Test + public void testLowVolumeLowConcurrency() throws Exception { + createTopic(TEST_TOPIC, 5, 2, 1); + + ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); + ArrayList readPartitions = new ArrayList(); + for (int i = 0; i < 5; i++) { + readPartitions.add(new Partition(TEST_TOPIC, i)); + } + final KafkaMessageListenerContainer kafkaMessageListenerContainer = new KafkaMessageListenerContainer(connectionFactory, readPartitions.toArray(new Partition[readPartitions.size()])); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(2); + + int expectedMessageCount = 100; + + final MutableListMultimap receivedData = new SynchronizedPutFastListMultimap(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + kafkaMessageListenerContainer.setMessageListener(new MessageListener() { + @Override + public void onMessage(KafkaMessage message) { + StringDecoder decoder = new StringDecoder(new VerifiableProperties()); + receivedData.put(message.getMetadata().getPartition().getId(),new KeyedMessageWithOffset(decodeKey(message, decoder), decodePayload(message, decoder), message.getMetadata().getOffset(), Thread.currentThread().getName(), message.getMetadata().getPartition().getId())); + latch.countDown(); + } + }); + + kafkaMessageListenerContainer.start(); + + createStringProducer(0).send(createMessages(100, TEST_TOPIC)); + + latch.await((expectedMessageCount/5000) + 1, TimeUnit.MINUTES); + kafkaMessageListenerContainer.stop(); + + assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); + assertThat(latch.getCount(), equalTo(0L)); + System.out.println("All messages received ... checking "); + + validateMessageReceipt(receivedData, 2, 5, 100, expectedMessageCount, readPartitions, 1); + + } + + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/OutboundTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/OutboundTests.java index 1471b3bac1..fe4ddb2948 100644 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/OutboundTests.java +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/OutboundTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2014 the original author or authors. + * Copyright 2015 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,18 +16,23 @@ package org.springframework.integration.kafka.outbound; -import static org.junit.Assert.assertNotNull; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThat; +import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import kafka.admin.AdminUtils; -import kafka.consumer.ConsumerConfig; +import kafka.api.OffsetRequest; +import kafka.common.TopicExistsException; import kafka.serializer.Decoder; import kafka.serializer.Encoder; - import org.junit.AfterClass; import org.junit.ClassRule; import org.junit.Test; @@ -35,26 +40,27 @@ import org.junit.Test; import org.springframework.context.expression.MapAccessor; import org.springframework.expression.spel.standard.SpelExpressionParser; import org.springframework.expression.spel.support.StandardEvaluationContext; +import org.springframework.integration.kafka.core.DefaultConnectionFactory; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.ZookeeperConfiguration; +import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer; +import org.springframework.integration.kafka.listener.MessageListener; +import org.springframework.integration.kafka.listener.MetadataStoreOffsetManager; import org.springframework.integration.kafka.rule.KafkaRunning; import org.springframework.integration.kafka.serializer.common.StringDecoder; import org.springframework.integration.kafka.serializer.common.StringEncoder; -import org.springframework.integration.kafka.support.ConsumerConfigFactoryBean; -import org.springframework.integration.kafka.support.ConsumerConfiguration; -import org.springframework.integration.kafka.support.ConsumerConnectionProvider; -import org.springframework.integration.kafka.support.ConsumerMetadata; -import org.springframework.integration.kafka.support.KafkaConsumerContext; import org.springframework.integration.kafka.support.KafkaHeaders; import org.springframework.integration.kafka.support.KafkaProducerContext; -import org.springframework.integration.kafka.support.MessageLeftOverTracker; import org.springframework.integration.kafka.support.ProducerConfiguration; import org.springframework.integration.kafka.support.ProducerFactoryBean; import org.springframework.integration.kafka.support.ProducerMetadata; import org.springframework.integration.kafka.support.ZookeeperConnect; -import org.springframework.messaging.Message; +import org.springframework.integration.kafka.util.MessageUtils; import org.springframework.messaging.support.MessageBuilder; /** * @author Gary Russell + * @author Marius Bogoevici * @since 1.0 * */ @@ -76,9 +82,42 @@ public class OutboundTests { @Test public void testAsyncProducerFlushed() throws Exception { - KafkaConsumerContext consumerContext = createConsumer(); - // pre-consume to start the receiver because the high-level API doesn't support --from-beginning - consumerContext.receive(); + + // create the topic + + try { + AdminUtils.createTopic(kafkaRunning.getZkClient(), TOPIC, 1, 1, new Properties()); + } + catch (TopicExistsException e) { + // do nothing + } + + final String suffix = UUID.randomUUID().toString(); + + ZookeeperConfiguration configuration = new ZookeeperConfiguration(new ZookeeperConnect()); + DefaultConnectionFactory connectionFactory = new DefaultConnectionFactory(configuration); + connectionFactory.afterPropertiesSet(); + final KafkaMessageListenerContainer kafkaMessageListenerContainer = new KafkaMessageListenerContainer(connectionFactory, TOPIC); + kafkaMessageListenerContainer.setMaxFetch(100); + kafkaMessageListenerContainer.setConcurrency(1); + MetadataStoreOffsetManager offsetManager = new MetadataStoreOffsetManager(connectionFactory); + // start reading at the end of the + offsetManager.setReferenceTimestamp(OffsetRequest.LatestTime()); + kafkaMessageListenerContainer.setOffsetManager(offsetManager); + final Decoder decoder = new StringDecoder(); + + int expectedMessageCount = 2; + final List payloads = new ArrayList(); + final CountDownLatch latch = new CountDownLatch(expectedMessageCount); + kafkaMessageListenerContainer.setMessageListener(new MessageListener() { + @Override + public void onMessage(KafkaMessage message) { + payloads.add(MessageUtils.decodePayload(message, decoder)); + latch.countDown(); + } + }); + + kafkaMessageListenerContainer.start(); KafkaProducerContext kafkaProducerContext = new KafkaProducerContext(); ProducerMetadata producerMetadata = new ProducerMetadata(TOPIC); @@ -97,7 +136,7 @@ public class OutboundTests { kafkaProducerContext.setProducerConfigurations(Collections.singletonMap(TOPIC, config)); KafkaProducerMessageHandler handler = new KafkaProducerMessageHandler(kafkaProducerContext); - handler.handleMessage(MessageBuilder.withPayload("foo") + handler.handleMessage(MessageBuilder.withPayload("foo"+suffix) .setHeader(KafkaHeaders.MESSAGE_KEY, "3") .setHeader(KafkaHeaders.TOPIC, TOPIC) .build()); @@ -108,45 +147,19 @@ public class OutboundTests { StandardEvaluationContext evaluationContext = new StandardEvaluationContext(); evaluationContext.addPropertyAccessor(new MapAccessor()); handler.setIntegrationEvaluationContext(evaluationContext); - handler.handleMessage(MessageBuilder.withPayload("bar") + handler.handleMessage(MessageBuilder.withPayload("bar"+suffix) .setHeader("foo", "3") .setHeader("bar", TOPIC) .build()); kafkaProducerContext.stop(); - Message>>> received = consumerContext.receive(); - assertNotNull(received); - if (((Map) received.getPayload()).size() < 2) { - received = consumerContext.receive(); - assertNotNull(received); + latch.await(1000, TimeUnit.MILLISECONDS); + assertThat(latch.getCount(), equalTo(0L)); + for (String payload : payloads) { + assertThat(payload, endsWith(suffix)); } - consumerContext.destroy(); - } - - private KafkaConsumerContext createConsumer() throws Exception { - KafkaConsumerContext consumerContext = new KafkaConsumerContext(); - ZookeeperConnect zookeeperConnect = new ZookeeperConnect(); - zookeeperConnect.setZkConnect("localhost:2181"); - consumerContext.setZookeeperConnect(zookeeperConnect); - ConsumerMetadata consumerMetadata = new ConsumerMetadata(); - consumerMetadata.setGroupId("foo"); - Decoder decoder = new StringDecoder(); - consumerMetadata.setValueDecoder(decoder); - consumerMetadata.setKeyDecoder(decoder); - consumerMetadata.setTopicStreamMap(Collections.singletonMap(TOPIC, 1)); - Properties consumerProps = new Properties(); - consumerProps.put("consumer.timeout.ms", "500"); - ConsumerConfigFactoryBean consumerConfigFactoryBean = new ConsumerConfigFactoryBean( - consumerMetadata, zookeeperConnect, consumerProps); - ConsumerConfig consumerConfig = consumerConfigFactoryBean.getObject(); - ConsumerConnectionProvider consumerConnectionProvider = new ConsumerConnectionProvider(consumerConfig); - MessageLeftOverTracker messageLeftOverTracker = new MessageLeftOverTracker(); - ConsumerConfiguration cConfig = new ConsumerConfiguration(consumerMetadata, - consumerConnectionProvider, messageLeftOverTracker); - cConfig.setMaxMessages(1); - consumerContext.setConsumerConfigurations(Collections.singletonMap("foo", cConfig)); - return consumerContext; + kafkaMessageListenerContainer.stop(); } } diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaEmbedded.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaEmbedded.java new file mode 100644 index 0000000000..1ade044eef --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaEmbedded.java @@ -0,0 +1,177 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.rule; + +import static scala.collection.JavaConversions.asScalaBuffer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.impl.list.mutable.FastList; +import com.gs.collections.impl.utility.ListIterate; +import kafka.Kafka; +import kafka.server.KafkaConfig; +import kafka.server.KafkaServer; +import kafka.utils.SystemTime$; +import kafka.utils.TestUtils; +import kafka.utils.TestZKUtils; +import kafka.utils.Utils; +import kafka.utils.ZKStringSerializer$; +import kafka.zk.EmbeddedZookeeper; +import org.I0Itec.zkclient.ZkClient; +import org.I0Itec.zkclient.exception.ZkInterruptedException; +import org.junit.rules.ExternalResource; +import scala.collection.JavaConversions; + +import org.springframework.integration.kafka.core.BrokerAddress; + +/** + * @author Marius Bogoevici + */ +@SuppressWarnings("serial") +public class KafkaEmbedded extends ExternalResource implements KafkaRule { + + private int count; + + private boolean controlledShutdown; + + private List kafkaPorts; + + private List kafkaServers; + + private EmbeddedZookeeper zookeeper; + + private ZkClient zookeeperClient; + + @SuppressWarnings("unchecked") + public KafkaEmbedded(int count, boolean controlledShutdown) { + this.count = count; + this.controlledShutdown = controlledShutdown; + this.kafkaPorts = JavaConversions.asJavaList((scala.collection.immutable.List) TestUtils.choosePorts(count)); + } + + public KafkaEmbedded(int count) { + this(count, false); + } + + @Override + protected void before() throws Throwable { + zookeeper = new EmbeddedZookeeper(TestZKUtils.zookeeperConnect()); + int zkConnectionTimeout = 6000; + int zkSessionTimeout = 6000; + zookeeperClient = new ZkClient(TestZKUtils.zookeeperConnect(), zkSessionTimeout, zkConnectionTimeout, ZKStringSerializer$.MODULE$); + kafkaServers = new ArrayList(); + for (int i = 0; i < count; i++) { + Properties brokerConfigProperties = TestUtils.createBrokerConfig(i, kafkaPorts.get(i)); + brokerConfigProperties.put("controlled.shutdown.enable", Boolean.toString(controlledShutdown)); + KafkaServer server = TestUtils.createServer(new KafkaConfig(brokerConfigProperties), SystemTime$.MODULE$); + kafkaServers.add(server); + } + } + + @Override + protected void after() { + for (KafkaServer kafkaServer : kafkaServers) { + try { + kafkaServer.shutdown(); + } + catch (Exception e) { + // do nothing + } + try { + Utils.rm(kafkaServer.config().logDirs()); + } + catch (Exception e) { + // do nothing + } + } + try { + zookeeperClient.close(); + } + catch (ZkInterruptedException e) { + // do nothing + } + try { + zookeeper.shutdown(); + } + catch (Exception e) { + // do nothing + } + } + + public List getKafkaServers() { + return kafkaServers; + } + + public KafkaServer getKafkaServer(int id) { + return kafkaServers.get(id); + } + + public EmbeddedZookeeper getZookeeper() { + return zookeeper; + } + + @Override + public ZkClient getZkClient() { + return zookeeperClient; + } + + public String getZookeeperConnectionString() { + return zookeeper.connectString(); + } + + @Override + public List getBrokerAddresses() { + return ListIterate.collect(kafkaServers, new Function() { + @Override + public BrokerAddress valueOf(KafkaServer kafkaServer) { + return new BrokerAddress(kafkaServer.config().hostName(), kafkaServer.config().port()); + } + }); + } + + public void bounce(BrokerAddress brokerAddress) { + for (KafkaServer kafkaServer : getKafkaServers()) { + if (brokerAddress.equals(new BrokerAddress(kafkaServer.config().hostName(), kafkaServer.config().port()))) { + kafkaServer.shutdown(); + } + } + } + + public void bounce(int index) { + kafkaServers.get(index).shutdown(); + TestUtils.waitUntilMetadataIsPropagated(asScalaBuffer(kafkaServers), "test-topic", 0, 5000L); + } + + @Override + public String getBrokersAsString() { + return FastList.newList(getBrokerAddresses()).collect(new Function() { + @Override + public String valueOf(BrokerAddress object) { + return object.getHost() + ":" + object.getPort(); + } + }).makeString(","); + } + + @Override + public boolean isEmbedded() { + return true; + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRule.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRule.java new file mode 100644 index 0000000000..9b6bac37d2 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRule.java @@ -0,0 +1,44 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.integration.kafka.rule; + +import java.util.List; + +import kafka.server.KafkaServer; +import org.I0Itec.zkclient.ZkClient; +import org.junit.rules.TestRule; + +import org.springframework.integration.kafka.core.BrokerAddress; + +/** + * Common functionality for the Kafka JUnit rules + * + * @author Marius Bogoevici + */ +public interface KafkaRule extends TestRule { + + ZkClient getZkClient(); + + List getBrokerAddresses(); + + String getBrokersAsString(); + + boolean isEmbedded(); + + List getKafkaServers(); +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRunning.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRunning.java index fdb4495b2b..e71dd0b663 100644 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRunning.java +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRunning.java @@ -15,6 +15,16 @@ */ package org.springframework.integration.kafka.rule; +import java.util.List; + +import com.gs.collections.api.block.function.Function; +import com.gs.collections.impl.block.factory.Functions; +import com.gs.collections.impl.list.mutable.FastList; +import com.gs.collections.impl.utility.ListIterate; +import kafka.cluster.Broker; +import kafka.server.KafkaServer; +import kafka.utils.ZKStringSerializer$; +import kafka.utils.ZkUtils; import org.I0Itec.zkclient.ZkClient; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -22,33 +32,33 @@ import org.junit.Assume; import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.junit.runners.model.Statement; +import scala.collection.JavaConversions; +import scala.collection.Seq; +import org.springframework.integration.kafka.core.BrokerAddress; import org.springframework.integration.kafka.core.ZookeeperConnectDefaults; -import kafka.utils.ZKStringSerializer$; -import kafka.utils.ZkUtils; - /** - *

- * A rule that prevents integration tests from failing if the Kafka server is not running or not + * * A rule that prevents integration tests from failing if the Kafka server is not running or not * accessible. If the Kafka server is not running in the background all the tests here will simply be skipped because * of a violated assumption (showing as successful). - *

* The rule can be declared as static so that it only has to check once for all tests in the enclosing test case, but * there isn't a lot of overhead in making it non-static. * * @author Dave Syer * @author Artem Bilan * @author Gary Russell - * + * @author Marius Bogoevici * @since 1.0 */ -public class KafkaRunning extends TestWatcher { +public class KafkaRunning extends TestWatcher implements KafkaRule { private static final String ZOOKEEPER_CONNECT_STRING = ZookeeperConnectDefaults.ZK_CONNECT; private static final Log logger = LogFactory.getLog(KafkaRunning.class); + private ZkClient zkClient; + /** * @return a new rule that assumes an existing running broker */ @@ -56,17 +66,42 @@ public class KafkaRunning extends TestWatcher { return new KafkaRunning(); } - private ZkClient zkClient; - public ZkClient getZkClient() { return zkClient; } + @Override + @SuppressWarnings("serial") + public List getBrokerAddresses() { + Seq allBrokersInCluster = ZkUtils.getAllBrokersInCluster(zkClient); + return ListIterate.collect(JavaConversions.asJavaList(allBrokersInCluster), new Function() { + @Override + public BrokerAddress valueOf(Broker broker) { + return new BrokerAddress(broker.host(), broker.port()); + } + }); + } + + @Override + public String getBrokersAsString() { + return FastList.newList(getBrokerAddresses()).collect(Functions.getToString()).makeString(","); + } + + @Override + public boolean isEmbedded() { + return false; + } + + @Override + public List getKafkaServers() { + throw new UnsupportedOperationException("Not supported on the external rule"); + } + @Override public Statement apply(Statement base, Description description) { try { this.zkClient = new ZkClient(ZOOKEEPER_CONNECT_STRING, 1000, 1000, ZKStringSerializer$.MODULE$); - if (ZkUtils.getAllBrokersInCluster(zkClient).size() == 0) { + if (getBrokerAddresses().size() == 0) { throw new IllegalStateException("No running Kafka brokers"); } }