From 9e626cb18b3f583a24a54703f82c3f55a1369321 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Mon, 18 Mar 2013 23:31:53 -0400 Subject: [PATCH] INTEXT-44 - Add Kafka Support * Add Inbound adapter * Add Outbound adapter * Add serialization * Initial Documentation * Add unit tests * Add async producer * Support Kafka 0.8 * Add Kafka Sample See: https://jira.springsource.org/browse/INTEXT-44 --- .../src/api/overview.html | 22 + spring-integration-kafka/src/dist/license.txt | 201 ++++++++ spring-integration-kafka/src/dist/notice.txt | 21 + .../xml/KafkaConsumerContextParser.java | 122 +++++ .../xml/KafkaInboundChannelAdapterParser.java | 49 ++ .../config/xml/KafkaNamespaceHandler.java | 38 ++ .../KafkaOutboundChannelAdapterParser.java | 45 ++ .../xml/KafkaProducerContextParser.java | 89 ++++ .../config/xml/ZookeeperConnectParser.java | 46 ++ .../kafka/config/xml/package-info.java | 4 + .../kafka/core/KafkaConsumerDefaults.java | 40 ++ .../kafka/core/ZookeeperConnectDefaults.java | 31 ++ .../integration/kafka/core/package-info.java | 4 + .../KafkaHighLevelConsumerMessageSource.java | 47 ++ .../kafka/inbound/package-info.java | 4 + .../outbound/KafkaProducerMessageHandler.java | 41 ++ .../integration/kafka/package-info.java | 4 + .../avro/AvroBackedKafkaDecoder.java | 54 +++ .../avro/AvroBackedKafkaEncoder.java | 52 ++ .../kafka/serializer/avro/AvroSerializer.java | 52 ++ .../avro/AvroSpecificDatumSerializer.java | 52 ++ .../serializer/common/StringEncoder.java | 41 ++ .../support/ConsumerConfigFactoryBean.java | 59 +++ .../kafka/support/ConsumerConfiguration.java | 209 ++++++++ .../support/ConsumerConnectionProvider.java | 35 ++ .../kafka/support/ConsumerMetadata.java | 174 +++++++ .../kafka/support/DefaultPartitioner.java | 36 ++ .../kafka/support/KafkaConsumerContext.java | 76 +++ .../kafka/support/KafkaProducerContext.java | 63 +++ .../kafka/support/MessageLeftOverTracker.java | 44 ++ .../kafka/support/ProducerConfiguration.java | 102 ++++ .../kafka/support/ProducerFactoryBean.java | 74 +++ .../kafka/support/ProducerMetadata.java | 139 ++++++ .../kafka/support/ZookeeperConnect.java | 60 +++ .../kafka/support/package-info.java | 4 + .../main/resources/META-INF/spring.handlers | 1 + .../main/resources/META-INF/spring.schemas | 2 + .../main/resources/META-INF/spring.tooling | 4 + .../xml/spring-integration-kafka-1.0.xsd | 451 ++++++++++++++++++ .../config/xml/spring-integration-kafka.gif | Bin 0 -> 572 bytes .../docbook/SIAdapterLowerPrefix.xml | 72 +++ .../src/reference/docbook/history.xml | 4 + .../src/reference/docbook/images/logo.png | Bin 0 -> 9627 bytes .../src/reference/docbook/index.xml | 67 +++ .../src/reference/docbook/resources.xml | 15 + .../src/reference/docbook/whats-new.xml | 8 + .../xml/KafkaConsumerContextParserTests.java | 47 ++ .../xml/KafkaInboundAdapterParserTests.java | 45 ++ .../xml/KafkaOutboundAdapterParserTests.java | 46 ++ .../xml/KafkaProducerContextParserTests.java | 73 +++ .../xml/ZookeeperConnectParserTests.java | 59 +++ .../AvroBackedKafkaSerializerTest.java | 59 +++ .../support/ConsumerConfigurationTests.java | 306 ++++++++++++ .../support/KafkaConsumerContextTest.java | 80 ++++ .../support/ProducerConfigurationTests.java | 303 ++++++++++++ .../support/ProducerFactoryBeanTests.java | 63 +++ .../test/utils/NonSerializableTestKey.java | 35 ++ .../utils/NonSerializableTestPayload.java | 34 ++ .../integration/kafka/test/utils/TestKey.java | 38 ++ .../kafka/test/utils/TestObject.java | 38 ++ .../kafka/test/utils/TestPayload.java | 38 ++ .../src/test/resources/log4j.properties | 8 + ...afkaConsumerContextParserTests-context.xml | 30 ++ .../xml/kafkaInboundAdapterCommon-context.xml | 17 + ...kafkaInboundAdapterParserTests-context.xml | 40 ++ ...afkaOutboundAdapterParserTests-context.xml | 44 ++ ...afkaProducerContextParserTests-context.xml | 26 + .../zookeeperConnectParserTests-context.xml | 14 + 68 files changed, 4201 insertions(+) create mode 100644 spring-integration-kafka/src/api/overview.html create mode 100644 spring-integration-kafka/src/dist/license.txt create mode 100644 spring-integration-kafka/src/dist/notice.txt create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParser.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaInboundChannelAdapterParser.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaNamespaceHandler.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaOutboundChannelAdapterParser.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParser.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParser.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/package-info.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConnectDefaults.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/package-info.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaHighLevelConsumerMessageSource.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/package-info.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/outbound/KafkaProducerMessageHandler.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/package-info.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroBackedKafkaDecoder.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroBackedKafkaEncoder.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSerializer.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumSerializer.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringEncoder.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfigFactoryBean.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfiguration.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConnectionProvider.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerMetadata.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultPartitioner.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaConsumerContext.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaProducerContext.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/MessageLeftOverTracker.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerConfiguration.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerFactoryBean.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerMetadata.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ZookeeperConnect.java create mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/package-info.java create mode 100644 spring-integration-kafka/src/main/resources/META-INF/spring.handlers create mode 100644 spring-integration-kafka/src/main/resources/META-INF/spring.schemas create mode 100644 spring-integration-kafka/src/main/resources/META-INF/spring.tooling create mode 100644 spring-integration-kafka/src/main/resources/org/springframework/integration/config/xml/spring-integration-kafka-1.0.xsd create mode 100644 spring-integration-kafka/src/main/resources/org/springframework/integration/config/xml/spring-integration-kafka.gif create mode 100644 spring-integration-kafka/src/reference/docbook/SIAdapterLowerPrefix.xml create mode 100644 spring-integration-kafka/src/reference/docbook/history.xml create mode 100644 spring-integration-kafka/src/reference/docbook/images/logo.png create mode 100644 spring-integration-kafka/src/reference/docbook/index.xml create mode 100644 spring-integration-kafka/src/reference/docbook/resources.xml create mode 100644 spring-integration-kafka/src/reference/docbook/whats-new.xml create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/serializer/AvroBackedKafkaSerializerTest.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ConsumerConfigurationTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaConsumerContextTest.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerConfigurationTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerFactoryBeanTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/NonSerializableTestKey.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/NonSerializableTestPayload.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestKey.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestObject.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestPayload.java create mode 100644 spring-integration-kafka/src/test/resources/log4j.properties create mode 100644 spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaConsumerContextParserTests-context.xml create mode 100644 spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterCommon-context.xml create mode 100644 spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterParserTests-context.xml create mode 100644 spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaOutboundAdapterParserTests-context.xml create mode 100644 spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaProducerContextParserTests-context.xml create mode 100644 spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/zookeeperConnectParserTests-context.xml diff --git a/spring-integration-kafka/src/api/overview.html b/spring-integration-kafka/src/api/overview.html new file mode 100644 index 0000000000..d9e5faadd2 --- /dev/null +++ b/spring-integration-kafka/src/api/overview.html @@ -0,0 +1,22 @@ + + +This document is the API specification for Spring Integration Kafka Extension +
+
+

+ For further API reference and developer documentation, see the + Spring + Integration reference documentation. + That documentation contains more detailed, developer-targeted + descriptions, with conceptual overviews, definitions of terms, + workarounds, and working code examples. +

+ +

+ If you are interested in commercial training, consultancy, and + support for Spring Integration, please visit + http://www.springsource.com +

+
+ + diff --git a/spring-integration-kafka/src/dist/license.txt b/spring-integration-kafka/src/dist/license.txt new file mode 100644 index 0000000000..c94ec8522f --- /dev/null +++ b/spring-integration-kafka/src/dist/license.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by testData1) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class testData1 and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [testData1 of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/spring-integration-kafka/src/dist/notice.txt b/spring-integration-kafka/src/dist/notice.txt new file mode 100644 index 0000000000..f62045a212 --- /dev/null +++ b/spring-integration-kafka/src/dist/notice.txt @@ -0,0 +1,21 @@ + ======================================================================== + == NOTICE file corresponding to section 4 d of the Apache License, == + == Version 2.0, in this case for the Spring Integration distribution. == + ======================================================================== + + This product includes software developed by + the Apache Software Foundation (http://www.apache.org). + + The end-user documentation included with a redistribution, if any, + must include the following acknowledgement: + + "This product includes software developed by the Spring Framework + Project (http://www.springframework.org)." + + Alternatively, this acknowledgement may appear in the software itself, + if and wherever such third-party acknowledgements normally appear. + + The names "Spring", "Spring Framework", and "Spring Integration" must + not be used to endorse or promote products derived from this software + without prior written permission. For written permission, please contact + enquiries@springsource.com. diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParser.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParser.java new file mode 100644 index 0000000000..3eac811716 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParser.java @@ -0,0 +1,122 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.config.xml; + +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.beans.factory.config.BeanDefinitionHolder; +import org.springframework.beans.factory.support.AbstractBeanDefinition; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.xml.AbstractSingleBeanDefinitionParser; +import org.springframework.beans.factory.xml.ParserContext; +import org.springframework.integration.config.xml.IntegrationNamespaceUtils; +import org.springframework.integration.kafka.support.ConsumerConfigFactoryBean; +import org.springframework.integration.kafka.support.ConsumerConfiguration; +import org.springframework.integration.kafka.support.ConsumerConnectionProvider; +import org.springframework.integration.kafka.support.ConsumerMetadata; +import org.springframework.integration.kafka.support.KafkaConsumerContext; +import org.springframework.integration.kafka.support.MessageLeftOverTracker; +import org.springframework.util.StringUtils; +import org.springframework.util.xml.DomUtils; +import org.w3c.dom.Element; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author Soby Chacko + */ +public class KafkaConsumerContextParser extends AbstractSingleBeanDefinitionParser { + + @Override + protected Class getBeanClass(final Element element) { + return KafkaConsumerContext.class; + } + + @Override + protected void doParse(final Element element, final ParserContext parserContext, final BeanDefinitionBuilder builder) { + super.doParse(element, parserContext, builder); + + final Element consumerConfigurations = DomUtils.getChildElementByTagName(element, "consumer-configurations"); + parseConsumerConfigurations(consumerConfigurations, parserContext, builder, element); + } + + private void parseConsumerConfigurations(final Element consumerConfigurations, final ParserContext parserContext, + final BeanDefinitionBuilder builder, final Element parentElem) { + for (final Element consumerConfiguration : DomUtils.getChildElementsByTagName(consumerConfigurations, "consumer-configuration")) { + final BeanDefinitionBuilder consumerConfigurationBuilder = BeanDefinitionBuilder.genericBeanDefinition(ConsumerConfiguration.class); + final BeanDefinitionBuilder consumerMetadataBuilder = BeanDefinitionBuilder.genericBeanDefinition(ConsumerMetadata.class); + + IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration, "group-id"); + + IntegrationNamespaceUtils.setReferenceIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration, "value-decoder"); + IntegrationNamespaceUtils.setReferenceIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration, "key-decoder"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration, "key-class-type"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration, "value-class-type"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerConfigurationBuilder, consumerConfiguration, "max-messages"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerMetadataBuilder, parentElem, "consumer-timeout"); + + final Map topicStreamsMap = new HashMap(); + + for (final Element topicConfiguration : DomUtils.getChildElementsByTagName(consumerConfiguration, "topic")) { + final String topic = topicConfiguration.getAttribute("id"); + final String streams = topicConfiguration.getAttribute("streams"); + final Integer streamsInt = Integer.valueOf(streams); + topicStreamsMap.put(topic, streamsInt); + } + + consumerMetadataBuilder.addPropertyValue("topicStreamMap", topicStreamsMap); + + final BeanDefinition consumerMetadataBeanDef = consumerMetadataBuilder.getBeanDefinition(); + registerBeanDefinition(new BeanDefinitionHolder(consumerMetadataBeanDef, "consumerMetadata_" + consumerConfiguration.getAttribute("group-id")), + parserContext.getRegistry()); + + final String zookeeperConnectBean = parentElem.getAttribute("zookeeper-connect"); + IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, parentElem, zookeeperConnectBean); + + final BeanDefinitionBuilder consumerConfigFactoryBuilder = BeanDefinitionBuilder.genericBeanDefinition(ConsumerConfigFactoryBean.class); + consumerConfigFactoryBuilder.addConstructorArgReference("consumerMetadata_" + consumerConfiguration.getAttribute("group-id")); + + if (StringUtils.hasText(zookeeperConnectBean)) { + consumerConfigFactoryBuilder.addConstructorArgReference(zookeeperConnectBean); + } + + final BeanDefinition consumerConfigFactoryBuilderBeanDefinition = consumerConfigFactoryBuilder.getBeanDefinition(); + registerBeanDefinition(new BeanDefinitionHolder(consumerConfigFactoryBuilderBeanDefinition, "consumerConfigFactory_" + consumerConfiguration.getAttribute("group-id")), parserContext.getRegistry()); + + final BeanDefinitionBuilder consumerConnectionProviderBuilder = BeanDefinitionBuilder.genericBeanDefinition(ConsumerConnectionProvider.class); + consumerConnectionProviderBuilder.addConstructorArgReference("consumerConfigFactory_" + consumerConfiguration.getAttribute("group-id")); + + final BeanDefinition consumerConnectionProviderBuilderBeanDefinition = consumerConnectionProviderBuilder.getBeanDefinition(); + registerBeanDefinition(new BeanDefinitionHolder(consumerConnectionProviderBuilderBeanDefinition, "consumerConnectionProvider_" + consumerConfiguration.getAttribute("group-id")), parserContext.getRegistry()); + + + final BeanDefinitionBuilder messageLeftOverBeanDefinitionBuilder = BeanDefinitionBuilder.genericBeanDefinition(MessageLeftOverTracker.class); + final BeanDefinition messageLeftOverBeanDefinition = messageLeftOverBeanDefinitionBuilder.getBeanDefinition(); + registerBeanDefinition(new BeanDefinitionHolder(messageLeftOverBeanDefinition, "messageLeftOver_" + consumerConfiguration.getAttribute("group-id")), + parserContext.getRegistry()); + + consumerConfigurationBuilder.addConstructorArgReference("consumerMetadata_" + consumerConfiguration.getAttribute("group-id")); + consumerConfigurationBuilder.addConstructorArgReference("consumerConnectionProvider_" + consumerConfiguration.getAttribute("group-id")); + consumerConfigurationBuilder.addConstructorArgReference("messageLeftOver_" + consumerConfiguration.getAttribute("group-id")); + + final AbstractBeanDefinition consumerConfigurationBeanDefinition = consumerConfigurationBuilder.getBeanDefinition(); + + final String consumerConfigurationBeanName = "consumerConfiguration_" + consumerConfiguration.getAttribute("group-id"); + registerBeanDefinition(new BeanDefinitionHolder(consumerConfigurationBeanDefinition, consumerConfigurationBeanName), + parserContext.getRegistry()); + } + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaInboundChannelAdapterParser.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaInboundChannelAdapterParser.java new file mode 100644 index 0000000000..cc52b1e60b --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaInboundChannelAdapterParser.java @@ -0,0 +1,49 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.config.xml; + +import org.springframework.beans.BeanMetadataElement; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.xml.ParserContext; +import org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser; +import org.springframework.integration.config.xml.IntegrationNamespaceUtils; +import org.springframework.integration.kafka.inbound.KafkaHighLevelConsumerMessageSource; +import org.springframework.util.StringUtils; +import org.w3c.dom.Element; + +/** + * The Kafka Inbound Channel adapter parser + * + * @author Soby Chacko + * + */ +public class KafkaInboundChannelAdapterParser extends AbstractPollingInboundChannelAdapterParser { + @Override + protected BeanMetadataElement parseSource(final Element element, final ParserContext parserContext) { + final BeanDefinitionBuilder highLevelConsumerMessageSourceBuilder = + BeanDefinitionBuilder.genericBeanDefinition(KafkaHighLevelConsumerMessageSource.class); + + IntegrationNamespaceUtils.setReferenceIfAttributeDefined(highLevelConsumerMessageSourceBuilder, element, "kafka-decoder"); + + final String kafkaConsumerContext = element.getAttribute("kafka-consumer-context-ref"); + + if (StringUtils.hasText(kafkaConsumerContext)) { + highLevelConsumerMessageSourceBuilder.addConstructorArgReference(kafkaConsumerContext); + } + + return highLevelConsumerMessageSourceBuilder.getBeanDefinition(); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaNamespaceHandler.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaNamespaceHandler.java new file mode 100644 index 0000000000..92d7823e0a --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaNamespaceHandler.java @@ -0,0 +1,38 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.config.xml; + +import org.springframework.integration.config.xml.AbstractIntegrationNamespaceHandler; + +/** + * The namespace handler for the Kafka namespace + * + * @author Soby Chacko + * + */ +public class KafkaNamespaceHandler extends AbstractIntegrationNamespaceHandler { + /* (non-Javadoc) + * @see org.springframework.beans.factory.xml.NamespaceHandler#init() + */ + @Override + public void init() { + registerBeanDefinitionParser("zookeeper-connect", new ZookeeperConnectParser()); + registerBeanDefinitionParser("inbound-channel-adapter", new KafkaInboundChannelAdapterParser()); + registerBeanDefinitionParser("outbound-channel-adapter", new KafkaOutboundChannelAdapterParser()); + registerBeanDefinitionParser("producer-context", new KafkaProducerContextParser()); + registerBeanDefinitionParser("consumer-context", new KafkaConsumerContextParser()); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaOutboundChannelAdapterParser.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaOutboundChannelAdapterParser.java new file mode 100644 index 0000000000..714dad3099 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaOutboundChannelAdapterParser.java @@ -0,0 +1,45 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.config.xml; + +import org.springframework.beans.factory.support.AbstractBeanDefinition; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.xml.ParserContext; +import org.springframework.integration.config.xml.AbstractOutboundChannelAdapterParser; +import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler; +import org.springframework.util.StringUtils; +import org.w3c.dom.Element; + +/** + * + * @author Soby Chacko + * + */ +public class KafkaOutboundChannelAdapterParser extends AbstractOutboundChannelAdapterParser { + @Override + protected AbstractBeanDefinition parseConsumer(final Element element, final ParserContext parserContext) { + final BeanDefinitionBuilder kafkaProducerMessageHandlerBuilder = + BeanDefinitionBuilder.genericBeanDefinition(KafkaProducerMessageHandler.class); + + final String kafkaServerBeanName = element.getAttribute("kafka-producer-context-ref"); + + if (StringUtils.hasText(kafkaServerBeanName)) { + kafkaProducerMessageHandlerBuilder.addConstructorArgReference(kafkaServerBeanName); + } + + return kafkaProducerMessageHandlerBuilder.getBeanDefinition(); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParser.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParser.java new file mode 100644 index 0000000000..02ed893d25 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParser.java @@ -0,0 +1,89 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.config.xml; + +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.beans.factory.config.BeanDefinitionHolder; +import org.springframework.beans.factory.support.AbstractBeanDefinition; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.xml.AbstractSimpleBeanDefinitionParser; +import org.springframework.beans.factory.xml.ParserContext; +import org.springframework.integration.config.xml.IntegrationNamespaceUtils; +import org.springframework.integration.kafka.support.KafkaProducerContext; +import org.springframework.integration.kafka.support.ProducerConfiguration; +import org.springframework.integration.kafka.support.ProducerFactoryBean; +import org.springframework.integration.kafka.support.ProducerMetadata; +import org.springframework.util.StringUtils; +import org.springframework.util.xml.DomUtils; +import org.w3c.dom.Element; + +/** + * @author Soby Chacko + */ +public class KafkaProducerContextParser extends AbstractSimpleBeanDefinitionParser { + @Override + protected Class getBeanClass(final Element element) { + return KafkaProducerContext.class; + } + + @Override + protected void doParse(final Element element, final ParserContext parserContext, final BeanDefinitionBuilder builder) { + super.doParse(element, parserContext, builder); + + final Element topics = DomUtils.getChildElementByTagName(element, "producer-configurations"); + parseProducerConfigurations(topics, parserContext); + } + + private void parseProducerConfigurations(final Element topics, final ParserContext parserContext) { + for (final Element producerConfiguration : DomUtils.getChildElementsByTagName(topics, "producer-configuration")){ + final BeanDefinitionBuilder producerConfigurationBuilder = BeanDefinitionBuilder.genericBeanDefinition(ProducerConfiguration.class); + + final BeanDefinitionBuilder producerMetadataBuilder = BeanDefinitionBuilder.genericBeanDefinition(ProducerMetadata.class); + producerMetadataBuilder.addConstructorArgValue(producerConfiguration.getAttribute("topic")); + IntegrationNamespaceUtils.setReferenceIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "value-encoder"); + IntegrationNamespaceUtils.setReferenceIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "key-encoder"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "key-class-type"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "value-class-type"); + IntegrationNamespaceUtils.setReferenceIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "partitioner"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "compression-codec"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "async"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, "batch-num-messages"); + + final BeanDefinition producerMetadataBeanDef = producerMetadataBuilder.getBeanDefinition(); + registerBeanDefinition(new BeanDefinitionHolder(producerMetadataBeanDef, "producerMetadata_" + producerConfiguration.getAttribute("topic")), + parserContext.getRegistry()); + + final BeanDefinitionBuilder producerFactoryBuilder = BeanDefinitionBuilder.genericBeanDefinition(ProducerFactoryBean.class); + producerFactoryBuilder.addConstructorArgReference("producerMetadata_" + producerConfiguration.getAttribute("topic")); + + final String brokerList = producerConfiguration.getAttribute("broker-list"); + if (StringUtils.hasText(brokerList)) { + producerFactoryBuilder.addConstructorArgValue(producerConfiguration.getAttribute("broker-list")); + } + + final BeanDefinition producerfactoryBeanDefinition = producerFactoryBuilder.getBeanDefinition(); + registerBeanDefinition(new BeanDefinitionHolder(producerfactoryBeanDefinition, "prodFactory_" + producerConfiguration.getAttribute("topic")), parserContext.getRegistry()); + + producerConfigurationBuilder.addConstructorArgReference("producerMetadata_" + producerConfiguration.getAttribute("topic")); + producerConfigurationBuilder.addConstructorArgReference("prodFactory_" + producerConfiguration.getAttribute("topic")); + + final AbstractBeanDefinition producerConfigurationBeanDefinition = producerConfigurationBuilder.getBeanDefinition(); + final String producerConfigurationBeanName = "producerConfiguration_" + producerConfiguration.getAttribute("topic"); + registerBeanDefinition(new BeanDefinitionHolder(producerConfigurationBeanDefinition, producerConfigurationBeanName), + parserContext.getRegistry()); + } + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParser.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParser.java new file mode 100644 index 0000000000..cac01e914d --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParser.java @@ -0,0 +1,46 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.config.xml; + +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.xml.AbstractSimpleBeanDefinitionParser; +import org.springframework.beans.factory.xml.BeanDefinitionParserDelegate; +import org.springframework.beans.factory.xml.ParserContext; +import org.springframework.integration.config.xml.IntegrationNamespaceUtils; +import org.springframework.integration.kafka.support.ZookeeperConnect; +import org.w3c.dom.Element; + +/** + * @author Soby Chacko + */ +public class ZookeeperConnectParser extends AbstractSimpleBeanDefinitionParser { + @Override + protected Class getBeanClass(final Element element) { + return ZookeeperConnect.class; + } + + @Override + protected void doParse(final Element element, final ParserContext parserContext, final BeanDefinitionBuilder builder) { + super.doParse(element, parserContext, builder); + + IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, + BeanDefinitionParserDelegate.SCOPE_ATTRIBUTE); + IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-connect"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-connection-timeout"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-session-timeout"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-sync-time"); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/package-info.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/package-info.java new file mode 100644 index 0000000000..3289f40710 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/package-info.java @@ -0,0 +1,4 @@ +/** + * Provides parser classes to provide Xml namespace support for the Kafka components. + */ +package org.springframework.integration.kafka.config.xml; diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java new file mode 100644 index 0000000000..212678f1e4 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java @@ -0,0 +1,40 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.core; + +/** + * Kafka adapter specific message headers. + * + * @author Soby Chacko + */ +public class KafkaConsumerDefaults { + //High level consumer + public static final String GROUP_ID = "groupid"; + public static final String SOCKET_TIMEOUT = "30000"; + public static final String SOCKET_BUFFER_SIZE = "64*1024"; + public static final String FETCH_SIZE = "300 * 1024"; + public static final String BACKOFF_INCREMENT = "1000"; + public static final String QUEUED_CHUNKS_MAX = "100"; + public static final String AUTO_COMMIT_ENABLE = "true"; + public static final String AUTO_COMMIT_INTERVAL = "10000"; + public static final String AUTO_OFFSET_RESET = "smallest"; + //Overriding the default value of -1, which will make the consumer to wait indefinitely + public static final String CONSUMER_TIMEOUT = "5000"; + public static final String REBALANCE_RETRIES_MAX = "4"; + + private KafkaConsumerDefaults() { + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConnectDefaults.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConnectDefaults.java new file mode 100644 index 0000000000..11138d1d63 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConnectDefaults.java @@ -0,0 +1,31 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.core; + +/** + * + * @author Soby Chacko + * + */ +public class ZookeeperConnectDefaults { + public static final String ZK_CONNECT = "localhost:2181"; + public static final String ZK_CONNECTION_TIMEOUT = "6000"; + public static final String ZK_SESSION_TIMEOUT = "6000"; + public static final String ZK_SYNC_TIME = "2000"; + + private ZookeeperConnectDefaults() { + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/package-info.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/package-info.java new file mode 100644 index 0000000000..f404ec831c --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/package-info.java @@ -0,0 +1,4 @@ +/** + * Provides core classes of the Kafka module. + */ +package org.springframework.integration.kafka.core; diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaHighLevelConsumerMessageSource.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaHighLevelConsumerMessageSource.java new file mode 100644 index 0000000000..acab37b754 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaHighLevelConsumerMessageSource.java @@ -0,0 +1,47 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.inbound; + +import org.springframework.integration.Message; +import org.springframework.integration.context.IntegrationObjectSupport; +import org.springframework.integration.core.MessageSource; +import org.springframework.integration.kafka.support.KafkaConsumerContext; + +import java.util.List; +import java.util.Map; + +/** + * @author Soby Chacko + * + */ +public class KafkaHighLevelConsumerMessageSource extends IntegrationObjectSupport implements MessageSource>>> { + + private final KafkaConsumerContext kafkaConsumerContext; + + public KafkaHighLevelConsumerMessageSource(final KafkaConsumerContext kafkaConsumerContext) { + this.kafkaConsumerContext = kafkaConsumerContext; + } + + @Override + public Message>>> receive() { + return kafkaConsumerContext.receive(); + } + + @Override + public String getComponentType() { + return "kafka:inbound-channel-adapter"; + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/package-info.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/package-info.java new file mode 100644 index 0000000000..2688d7b67d --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/package-info.java @@ -0,0 +1,4 @@ +/** + * Provides inbound Spring Integration Kafka components. + */ +package org.springframework.integration.kafka.inbound; diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/outbound/KafkaProducerMessageHandler.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/outbound/KafkaProducerMessageHandler.java new file mode 100644 index 0000000000..ee4923b405 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/outbound/KafkaProducerMessageHandler.java @@ -0,0 +1,41 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.outbound; + +import org.springframework.integration.Message; +import org.springframework.integration.handler.AbstractMessageHandler; +import org.springframework.integration.kafka.support.KafkaProducerContext; + +/** + * @author Soby Chacko + */ +public class KafkaProducerMessageHandler extends AbstractMessageHandler { + + private final KafkaProducerContext kafkaProducerContext; + + public KafkaProducerMessageHandler(final KafkaProducerContext kafkaProducerContext) { + this.kafkaProducerContext = kafkaProducerContext; + } + + public KafkaProducerContext getKafkaProducerContext() { + return kafkaProducerContext; + } + + @Override + protected void handleMessageInternal(final Message message) throws Exception { + kafkaProducerContext.send(message); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/package-info.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/package-info.java new file mode 100644 index 0000000000..484759047a --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/package-info.java @@ -0,0 +1,4 @@ +/** + * Root package of the Kafka Module. + */ +package org.springframework.integration.kafka; diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroBackedKafkaDecoder.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroBackedKafkaDecoder.java new file mode 100644 index 0000000000..f131659d59 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroBackedKafkaDecoder.java @@ -0,0 +1,54 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.serializer.avro; + + +import kafka.serializer.Decoder; +import org.apache.avro.Schema; +import org.apache.avro.reflect.ReflectData; + +import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * @author Soby Chacko + */ +public class AvroBackedKafkaDecoder implements Decoder { + private static final Log LOG = LogFactory.getLog(AvroBackedKafkaDecoder.class); + + private final Class clazz; + + public AvroBackedKafkaDecoder(final Class clazz) { + this.clazz = clazz; + } + + @Override + @SuppressWarnings("unchecked") + public T fromBytes(final byte[] bytes) { + final Schema schema = ReflectData.get().getSchema(clazz); + final AvroSerializer avroSerializer = new AvroSerializer(); + + try { + return (T) avroSerializer.deserialize(bytes, schema); + } catch (IOException e) { + LOG.error("Failed to decode byte array for schema: " + schema.getFullName(), e); + } + + return null; + } +} + diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroBackedKafkaEncoder.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroBackedKafkaEncoder.java new file mode 100644 index 0000000000..8bc426b5d0 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroBackedKafkaEncoder.java @@ -0,0 +1,52 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.serializer.avro; + +import kafka.serializer.Encoder; +import org.apache.avro.Schema; +import org.apache.avro.reflect.ReflectData; + +import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * @author Soby Chacko + */ +public class AvroBackedKafkaEncoder implements Encoder { + private static final Log LOG = LogFactory.getLog(AvroBackedKafkaEncoder.class); + + private final Class clazz; + + public AvroBackedKafkaEncoder(final Class clazz) { + this.clazz = clazz; + } + + @Override + @SuppressWarnings("unchecked") + public byte[] toBytes(final T source) { + final Schema schema = ReflectData.get().getSchema(clazz); + final AvroSerializer avroSerializer = new AvroSerializer(); + + try { + return avroSerializer.serialize(source, schema); + } catch (IOException e) { + LOG.error("Failed to encode source for schema: " + schema.getFullName()); + } + + return null; + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSerializer.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSerializer.java new file mode 100644 index 0000000000..c3e751c872 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSerializer.java @@ -0,0 +1,52 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.serializer.avro; + +import org.apache.avro.Schema; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.Decoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.reflect.ReflectDatumReader; +import org.apache.avro.reflect.ReflectDatumWriter; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * @author Soby Chacko + */ +public class AvroSerializer { + public T deserialize(final byte[] bytes, final Schema schema) throws IOException { + final Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null); + final DatumReader reader = new ReflectDatumReader(schema); + + return reader.read(null, decoder); + } + + public byte[] serialize(final T input, final Schema schema) throws IOException { + final DatumWriter writer = new ReflectDatumWriter(schema); + final ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + final Encoder encoder = EncoderFactory.get().binaryEncoder(stream, null); + writer.write(input, encoder); + encoder.flush(); + + return stream.toByteArray(); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumSerializer.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumSerializer.java new file mode 100644 index 0000000000..12fb60dd85 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumSerializer.java @@ -0,0 +1,52 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.serializer.avro; + +import org.apache.avro.Schema; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.Decoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificDatumWriter; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * @author Soby Chacko + */ +public class AvroSpecificDatumSerializer { + public T deserialize(final byte[] bytes, final Schema schema) throws IOException { + final Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null); + final DatumReader reader = new SpecificDatumReader(schema); + + return reader.read(null, decoder); + } + + public byte[] serialize(final T input, final Schema schema) throws IOException { + final DatumWriter writer = new SpecificDatumWriter(schema); + final ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + final Encoder encoder = EncoderFactory.get().binaryEncoder(stream, null); + writer.write(input, encoder); + encoder.flush(); + + return stream.toByteArray(); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringEncoder.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringEncoder.java new file mode 100644 index 0000000000..ce5e0f3eaa --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringEncoder.java @@ -0,0 +1,41 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.serializer.common; + +import kafka.serializer.Encoder; +import kafka.utils.VerifiableProperties; + +import java.util.Properties; + +/** + * @author Soby Chacko + */ +public class StringEncoder implements Encoder { + private String encoding = "UTF8"; + + public void setEncoding(final String encoding){ + this.encoding = encoding; + } + + @Override + public byte[] toBytes(final Object o) { + final Properties props = new Properties(); + props.put("serializer.encoding", encoding); + + final VerifiableProperties verifiableProperties = new VerifiableProperties(props); + return new kafka.serializer.StringEncoder(verifiableProperties).toBytes((String)o); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfigFactoryBean.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfigFactoryBean.java new file mode 100644 index 0000000000..bd27da7a68 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfigFactoryBean.java @@ -0,0 +1,59 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import kafka.consumer.ConsumerConfig; +import org.springframework.beans.factory.FactoryBean; + +import java.util.Properties; + +/** + * @author Soby Chacko + */ +public class ConsumerConfigFactoryBean implements FactoryBean { + + private final ConsumerMetadata consumerMetadata; + private final ZookeeperConnect zookeeperConnect; + + public ConsumerConfigFactoryBean(final ConsumerMetadata consumerMetadata, + final ZookeeperConnect zookeeperConnect){ + this.consumerMetadata = consumerMetadata; + this.zookeeperConnect = zookeeperConnect; + } + + @Override + public ConsumerConfig getObject() throws Exception { + final Properties properties = new Properties(); + properties.put("zookeeper.connect", zookeeperConnect.getZkConnect()); + properties.put("zookeeper.session.timeout.ms", zookeeperConnect.getZkSessionTimeout()); + properties.put("zookeeper.sync.time.ms", zookeeperConnect.getZkSyncTime()); + properties.put("auto.commit.interval.ms", consumerMetadata.getAutoCommitInterval()); + properties.put("consumer.timeout.ms", consumerMetadata.getConsumerTimeout()); + properties.put("group.id", consumerMetadata.getGroupId()); + + return new ConsumerConfig(properties); + } + + @Override + public Class getObjectType() { + return ConsumerConfig.class; + } + + @Override + public boolean isSingleton() { + return true; + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfiguration.java new file mode 100644 index 0000000000..c7bb878cbd --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfiguration.java @@ -0,0 +1,209 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import kafka.consumer.ConsumerTimeoutException; +import kafka.consumer.KafkaStream; +import kafka.javaapi.consumer.ConsumerConnector; +import kafka.message.MessageAndMetadata; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.integration.MessagingException; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + * @author Soby Chacko + */ +public class ConsumerConfiguration { + private static final Log LOGGER = LogFactory.getLog(ConsumerConfiguration.class); + + private final ConsumerMetadata consumerMetadata; + private final ConsumerConnectionProvider consumerConnectionProvider; + private final MessageLeftOverTracker messageLeftOverTracker; + private ConsumerConnector consumerConnector; + private volatile int count = 0; + private int maxMessages = 1; + + private ExecutorService executorService = Executors.newCachedThreadPool(); + + public ConsumerConfiguration(final ConsumerMetadata consumerMetadata, + final ConsumerConnectionProvider consumerConnectionProvider, + final MessageLeftOverTracker messageLeftOverTracker) { + this.consumerMetadata = consumerMetadata; + this.consumerConnectionProvider = consumerConnectionProvider; + this.messageLeftOverTracker = messageLeftOverTracker; + } + + public ConsumerMetadata getConsumerMetadata() { + return consumerMetadata; + } + + public Map>> receive() { + count = messageLeftOverTracker.getCurrentCount(); + + final List>> tasks = new LinkedList>>(); + final Object lock = new Object(); + + final Map>> consumerMap = getConsumerMapWithMessageStreams(); + for (final List> streams : consumerMap.values()) { + for (final KafkaStream stream : streams) { + tasks.add(new Callable>() { + @Override + public List call() throws Exception { + final List rawMessages = new ArrayList(); + try { + while (count < maxMessages) { + final MessageAndMetadata messageAndMetadata = stream.iterator().next(); + synchronized (lock) { + if (count < maxMessages) { + rawMessages.add(messageAndMetadata); + count++; + } else { + messageLeftOverTracker.addMessageAndMetadata(messageAndMetadata); + } + } + } + } catch (ConsumerTimeoutException cte) { + LOGGER.info("Consumer timed out"); + } + return rawMessages; + } + }); + } + } + + return executeTasks(tasks); + } + + private Map>> executeTasks(final List>> tasks) { + + final Map>> messages = new ConcurrentHashMap>>(); + messages.putAll(getLeftOverMessageMap()); + + try { + for (final Future> result : executorService.invokeAll(tasks)) { + if (!result.get().isEmpty()) { + final String topic = result.get().get(0).topic(); + if (!messages.containsKey(topic)) { + messages.put(topic, getPayload(result.get())); + } else { + + final Map> existingPayloadMap = messages.get(topic); + getPayload(result.get(), existingPayloadMap); + } + } + } + } catch (Exception e) { + throw new MessagingException("Consuming from Kafka failed", e); + } + + if (messages.isEmpty()) { + return null; + } + + return messages; + } + + private Map>> getLeftOverMessageMap() { + + final Map>> messages = new ConcurrentHashMap>>(); + + for (final MessageAndMetadata mamd : messageLeftOverTracker.getMessageLeftOverFromPreviousPoll()) { + final String topic = mamd.topic(); + + if (!messages.containsKey(topic)) { + final List l = new ArrayList(); + l.add(mamd); + messages.put(topic, getPayload(l)); + } else { + final Map> existingPayloadMap = messages.get(topic); + final List l = new ArrayList(); + l.add(mamd); + getPayload(l, existingPayloadMap); + } + } + messageLeftOverTracker.clearMessagesLeftOver(); + return messages; + } + + private Map> getPayload(final List messageAndMetadatas) { + final Map> payloadMap = new ConcurrentHashMap>(); + + for (final MessageAndMetadata messageAndMetadata : messageAndMetadatas) { + if (!payloadMap.containsKey(messageAndMetadata.partition())) { + final List payload = new ArrayList(); + payload.add(messageAndMetadata.message()); + payloadMap.put(messageAndMetadata.partition(), payload); + } else { + final List payload = payloadMap.get(messageAndMetadata.partition()); + payload.add(messageAndMetadata.message()); + } + + } + + return payloadMap; + } + + private void getPayload(final List messageAndMetadatas, final Map> existingPayloadMap) { + for (final MessageAndMetadata messageAndMetadata : messageAndMetadatas) { + if (!existingPayloadMap.containsKey(messageAndMetadata.partition())) { + final List payload = new ArrayList(); + payload.add(messageAndMetadata.message()); + existingPayloadMap.put(messageAndMetadata.partition(), payload); + } else { + final List payload = existingPayloadMap.get(messageAndMetadata.partition()); + payload.add(messageAndMetadata.message()); + } + } + } + + @SuppressWarnings("unchecked") + public Map>> getConsumerMapWithMessageStreams() { + if (consumerMetadata.getValueDecoder() != null) { + return getConsumerConnector().createMessageStreams( + consumerMetadata.getTopicStreamMap(), + consumerMetadata.getValueDecoder(), + consumerMetadata.getValueDecoder()); + } + + return getConsumerConnector().createMessageStreams(consumerMetadata.getTopicStreamMap()); + } + + public int getMaxMessages() { + return maxMessages; + } + + public void setMaxMessages(final int maxMessages) { + this.maxMessages = maxMessages; + } + + public ConsumerConnector getConsumerConnector() { + if (consumerConnector == null) { + consumerConnector = consumerConnectionProvider.getConsumerConnector(); + } + + return consumerConnector; + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConnectionProvider.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConnectionProvider.java new file mode 100644 index 0000000000..d2287fde63 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConnectionProvider.java @@ -0,0 +1,35 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import kafka.consumer.ConsumerConfig; +import kafka.javaapi.consumer.ConsumerConnector; + +/** + * @author Soby Chacko + */ +public class ConsumerConnectionProvider { + + private final ConsumerConfig consumerConfig; + + public ConsumerConnectionProvider(final ConsumerConfig consumerConfig) { + this.consumerConfig = consumerConfig; + } + + public ConsumerConnector getConsumerConnector() { + return kafka.consumer.Consumer.createJavaConsumerConnector(consumerConfig); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerMetadata.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerMetadata.java new file mode 100644 index 0000000000..eb4cac5418 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerMetadata.java @@ -0,0 +1,174 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import kafka.serializer.Decoder; +import org.springframework.integration.kafka.core.KafkaConsumerDefaults; + +import java.util.Map; + +/** + * @author Soby Chacko + */ +public class ConsumerMetadata { + + //High level consumer defaults + private String groupId = KafkaConsumerDefaults.GROUP_ID; + private String socketTimeout = KafkaConsumerDefaults.SOCKET_TIMEOUT; + private String socketBufferSize = KafkaConsumerDefaults.SOCKET_BUFFER_SIZE; + private String fetchSize = KafkaConsumerDefaults.FETCH_SIZE; + private String backoffIncrement = KafkaConsumerDefaults.BACKOFF_INCREMENT; + private String queuedChunksMax = KafkaConsumerDefaults.QUEUED_CHUNKS_MAX; + private String autoCommitEnable = KafkaConsumerDefaults.AUTO_COMMIT_ENABLE; + private String autoCommitInterval = KafkaConsumerDefaults.AUTO_COMMIT_INTERVAL; + private String autoOffsetReset = KafkaConsumerDefaults.AUTO_OFFSET_RESET; + private String rebalanceRetriesMax = KafkaConsumerDefaults.REBALANCE_RETRIES_MAX; + private String consumerTimeout = KafkaConsumerDefaults.CONSUMER_TIMEOUT; + + private String topic; + private int streams; + private Decoder valueDecoder; + private Decoder keyDecoder; + private Map topicStreamMap; + + public String getGroupId() { + return groupId; + } + + public void setGroupId(final String groupId) { + this.groupId = groupId; + } + + public String getSocketTimeout() { + return socketTimeout; + } + + public void setSocketTimeout(final String socketTimeout) { + this.socketTimeout = socketTimeout; + } + + public String getSocketBufferSize() { + return socketBufferSize; + } + + public void setSocketBufferSize(final String socketBufferSize) { + this.socketBufferSize = socketBufferSize; + } + + public String getFetchSize() { + return fetchSize; + } + + public void setFetchSize(final String fetchSize) { + this.fetchSize = fetchSize; + } + + public String getBackoffIncrement() { + return backoffIncrement; + } + + public void setBackoffIncrement(final String backoffIncrement) { + this.backoffIncrement = backoffIncrement; + } + + public String getQueuedChunksMax() { + return queuedChunksMax; + } + + public void setQueuedChunksMax(final String queuedChunksMax) { + this.queuedChunksMax = queuedChunksMax; + } + + public String getAutoCommitEnable() { + return autoCommitEnable; + } + + public void setAutoCommitEnable(final String autoCommitEnable) { + this.autoCommitEnable = autoCommitEnable; + } + + public String getAutoCommitInterval() { + return autoCommitInterval; + } + + public void setAutoCommitInterval(final String autoCommitInterval) { + this.autoCommitInterval = autoCommitInterval; + } + + public String getAutoOffsetReset() { + return autoOffsetReset; + } + + public void setAutoOffsetReset(final String autoOffsetReset) { + this.autoOffsetReset = autoOffsetReset; + } + + public String getRebalanceRetriesMax() { + return rebalanceRetriesMax; + } + + public void setRebalanceRetriesMax(final String rebalanceRetriesMax) { + this.rebalanceRetriesMax = rebalanceRetriesMax; + } + + public String getConsumerTimeout() { + return consumerTimeout; + } + + public void setConsumerTimeout(final String consumerTimeout) { + this.consumerTimeout = consumerTimeout; + } + + public String getTopic() { + return topic; + } + + public void setTopic(final String topic) { + this.topic = topic; + } + + public int getStreams() { + return streams; + } + + public void setStreams(final int streams) { + this.streams = streams; + } + + public Decoder getValueDecoder() { + return valueDecoder; + } + + public void setValueDecoder(final Decoder valueDecoder) { + this.valueDecoder = valueDecoder; + } + + public Decoder getKeyDecoder() { + return keyDecoder; + } + + public void setKeyDecoder(final Decoder keyDecoder) { + this.keyDecoder = keyDecoder; + } + + public Map getTopicStreamMap() { + return topicStreamMap; + } + + public void setTopicStreamMap(final Map topicStreamMap) { + this.topicStreamMap = topicStreamMap; + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultPartitioner.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultPartitioner.java new file mode 100644 index 0000000000..3147b9de9c --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultPartitioner.java @@ -0,0 +1,36 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import kafka.producer.Partitioner; +import kafka.utils.Utils; + +/** + * @author Soby Chacko + * + * This class is for internal use only and therefore is at default access level + */ +class DefaultPartitioner implements Partitioner { + /** + * Uses the key to calculate a partition bucket id for routing + * the data to the appropriate broker partition + * @return an integer between 0 and numPartitions-1 + */ + @Override + public int partition(final T key, final int numPartitions) { + return Utils.abs(key.hashCode()) % numPartitions; + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaConsumerContext.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaConsumerContext.java new file mode 100644 index 0000000000..a42e458403 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaConsumerContext.java @@ -0,0 +1,76 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.BeanFactoryAware; +import org.springframework.beans.factory.ListableBeanFactory; +import org.springframework.integration.Message; +import org.springframework.integration.kafka.core.KafkaConsumerDefaults; +import org.springframework.integration.support.MessageBuilder; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author Soby Chacko + */ +public class KafkaConsumerContext implements BeanFactoryAware { + private Map consumerConfigurations; + private String consumerTimeout = KafkaConsumerDefaults.CONSUMER_TIMEOUT; + private ZookeeperConnect zookeeperConnect; + + public Collection getConsumerConfigurations() { + return consumerConfigurations.values(); + } + + @Override + public void setBeanFactory(final BeanFactory beanFactory) throws BeansException { + consumerConfigurations = ((ListableBeanFactory) beanFactory).getBeansOfType(ConsumerConfiguration.class); + } + + public Message>>> receive() { + final Map>> consumedData = new HashMap>>(); + + for (final ConsumerConfiguration consumerConfiguration : getConsumerConfigurations()) { + final Map>> messages = consumerConfiguration.receive(); + + if (messages != null){ + consumedData.putAll(messages); + } + } + return MessageBuilder.withPayload(consumedData).build(); + } + + public String getConsumerTimeout() { + return consumerTimeout; + } + + public void setConsumerTimeout(final String consumerTimeout) { + this.consumerTimeout = consumerTimeout; + } + + public ZookeeperConnect getZookeeperConnect() { + return zookeeperConnect; + } + + public void setZookeeperConnect(final ZookeeperConnect zookeeperConnect) { + this.zookeeperConnect = zookeeperConnect; + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaProducerContext.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaProducerContext.java new file mode 100644 index 0000000000..8a815d598f --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaProducerContext.java @@ -0,0 +1,63 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.BeanFactoryAware; +import org.springframework.beans.factory.ListableBeanFactory; +import org.springframework.integration.Message; + +import java.util.Collection; +import java.util.Map; + +/** + * @author Soby Chacko + */ +public class KafkaProducerContext implements BeanFactoryAware { + private Map topicsConfiguration; + + @SuppressWarnings("unchecked") + public void send(final Message message) throws Exception { + final ProducerConfiguration producerConfiguration = + getTopicConfiguration(message.getHeaders().get("topic", String.class)); + + if (producerConfiguration != null) { + producerConfiguration.send(message); + } + } + + private ProducerConfiguration getTopicConfiguration(final String topic){ + final Collection topics = topicsConfiguration.values(); + + for (final ProducerConfiguration producerConfiguration : topics){ + if (producerConfiguration.getProducerMetadata().getTopic().equals(topic)){ + return producerConfiguration; + } + } + + return null; + } + + public Map getTopicsConfiguration() { + return topicsConfiguration; + } + + @Override + public void setBeanFactory(final BeanFactory beanFactory) throws BeansException { + topicsConfiguration = ((ListableBeanFactory)beanFactory).getBeansOfType(ProducerConfiguration.class); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/MessageLeftOverTracker.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/MessageLeftOverTracker.java new file mode 100644 index 0000000000..a7d23e0e73 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/MessageLeftOverTracker.java @@ -0,0 +1,44 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import kafka.message.MessageAndMetadata; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author Soby Chacko + */ +public class MessageLeftOverTracker { + private final List messageLeftOverFromPreviousPoll = new ArrayList(); + + public void addMessageAndMetadata(final MessageAndMetadata messageAndMetadata){ + messageLeftOverFromPreviousPoll.add(messageAndMetadata); + } + + public List getMessageLeftOverFromPreviousPoll(){ + return messageLeftOverFromPreviousPoll; + } + + public void clearMessagesLeftOver(){ + messageLeftOverFromPreviousPoll.clear(); + } + + public int getCurrentCount() { + return messageLeftOverFromPreviousPoll.size(); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerConfiguration.java new file mode 100644 index 0000000000..93e5c6694a --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerConfiguration.java @@ -0,0 +1,102 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import kafka.javaapi.producer.Producer; +import kafka.producer.KeyedMessage; +import kafka.serializer.DefaultEncoder; +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.springframework.integration.Message; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; + +/** + * @author Soby Chacko + */ +public class ProducerConfiguration { + private final Producer producer; + private final ProducerMetadata producerMetadata; + + public ProducerConfiguration(final ProducerMetadata producerMetadata, final Producer producer){ + this.producerMetadata = producerMetadata; + this.producer = producer; + } + + public ProducerMetadata getProducerMetadata() { + return producerMetadata; + } + + public void send(final Message message) throws Exception { + final V v = getPayload(message); + + if (message.getHeaders().containsKey("messageKey")) { + producer.send(new KeyedMessage(producerMetadata.getTopic(), getKey(message), v)); + } else { + producer.send(new KeyedMessage(producerMetadata.getTopic(), v)); + } + } + + @SuppressWarnings("unchecked") + private V getPayload(final Message message) throws Exception { + if (producerMetadata.getValueEncoder().getClass().isAssignableFrom(DefaultEncoder.class)) { + return (V) getByteStream(message.getPayload()); + } else if (message.getPayload().getClass().isAssignableFrom(producerMetadata.getValueClassType())) { + return producerMetadata.getValueClassType().cast(message.getPayload()); + } + + throw new Exception("Message payload type is not matching with what is configured"); + } + + @SuppressWarnings("unchecked") + private K getKey(final Message message) throws Exception { + final Object key = message.getHeaders().get("messageKey"); + + if (producerMetadata.getKeyEncoder().getClass().isAssignableFrom(DefaultEncoder.class)) { + return (K) getByteStream(key); + } + + return message.getHeaders().get("messageKey", producerMetadata.getKeyClassType()); + } + + private static boolean isRawByteArray(final Object obj){ + return obj instanceof byte[]; + } + + private static byte[] getByteStream(final Object obj) throws IOException { + if (isRawByteArray(obj)){ + return (byte[])obj; + } + + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + final ObjectOutputStream os = new ObjectOutputStream(out); + os.writeObject(obj); + + return out.toByteArray(); + } + + @Override + public boolean equals(final Object obj){ + return EqualsBuilder.reflectionEquals(this, obj); + } + + @Override + public int hashCode() { + return HashCodeBuilder.reflectionHashCode(this); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerFactoryBean.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerFactoryBean.java new file mode 100644 index 0000000000..801c8e53f8 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerFactoryBean.java @@ -0,0 +1,74 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import kafka.javaapi.producer.Producer; +import kafka.producer.ProducerConfig; +import kafka.producer.ProducerPool; +import kafka.producer.async.DefaultEventHandler; +import kafka.producer.async.EventHandler; +import org.springframework.beans.factory.FactoryBean; +import scala.collection.mutable.HashMap; + +import java.util.Properties; + +/** + * @author Soby Chacko + */ +public class ProducerFactoryBean implements FactoryBean> { + + private final String brokerList; + private final ProducerMetadata producerMetadata; + + public ProducerFactoryBean(final ProducerMetadata producerMetadata, final String brokerList){ + this.producerMetadata = producerMetadata; + this.brokerList = brokerList; + } + + @Override + public Producer getObject() throws Exception { + final Properties props = new Properties(); + props.put("metadata.broker.list", brokerList); + props.put("compression.codec", producerMetadata.getCompressionCodec()); + + if (producerMetadata.isAsync()){ + props.put("producer.type", "async"); + if (producerMetadata.getBatchNumMessages() != null){ + props.put("batch.num.messages", producerMetadata.getBatchNumMessages()); + } + } + + final ProducerConfig config = new ProducerConfig(props); + final EventHandler eventHandler = new DefaultEventHandler(config, + producerMetadata.getPartitioner() == null ? new DefaultPartitioner() : producerMetadata.getPartitioner(), + producerMetadata.getValueEncoder(), producerMetadata.getKeyEncoder(), + new ProducerPool(config), new HashMap()); + + final kafka.producer.Producer prod = new kafka.producer.Producer(config, + eventHandler); + return new Producer(prod); + } + + @Override + public Class getObjectType() { + return Producer.class; + } + + @Override + public boolean isSingleton() { + return true; + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerMetadata.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerMetadata.java new file mode 100644 index 0000000000..a10a4fa5b4 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerMetadata.java @@ -0,0 +1,139 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import kafka.producer.Partitioner; +import kafka.serializer.DefaultEncoder; +import kafka.serializer.Encoder; +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.springframework.beans.factory.InitializingBean; + +/** + * @author Soby Chacko + */ +public class ProducerMetadata implements InitializingBean { + private Encoder keyEncoder; + private Encoder valueEncoder; + private Class keyClassType; + private Class valueClassType; + private final String topic; + private String compressionCodec = "default"; + private Partitioner partitioner; + private boolean async = false; + private String batchNumMessages; + + public ProducerMetadata(final String topic) { + this.topic = topic; + } + + public String getTopic() { + return topic; + } + + public Encoder getKeyEncoder() { + return keyEncoder; + } + + public void setKeyEncoder(final Encoder keyEncoder) { + this.keyEncoder = keyEncoder; + } + + public Encoder getValueEncoder() { + return valueEncoder; + } + + public void setValueEncoder(final Encoder valueEncoder) { + this.valueEncoder = valueEncoder; + } + + public Class getKeyClassType() { + return keyClassType; + } + + public void setKeyClassType(final Class keyClassType) { + this.keyClassType = keyClassType; + } + + public Class getValueClassType() { + return valueClassType; + } + + public void setValueClassType(final Class valueClassType) { + this.valueClassType = valueClassType; + } + + //TODO: Use an enum + public String getCompressionCodec() { + if (compressionCodec.equalsIgnoreCase("gzip")) { + return "1"; + } else if (compressionCodec.equalsIgnoreCase("snappy")) { + return "2"; + } + + return "0"; + } + + public void setCompressionCodec(final String compressionCodec) { + this.compressionCodec = compressionCodec; + } + + public Partitioner getPartitioner() { + return partitioner; + } + + public void setPartitioner(final Partitioner partitioner) { + this.partitioner = partitioner; + } + + @Override + @SuppressWarnings("unchecked") + public void afterPropertiesSet() throws Exception { + if (valueEncoder == null) { + setValueEncoder((Encoder) new DefaultEncoder(null)); + } + + if (keyEncoder == null) { + setKeyEncoder((Encoder) getValueEncoder()); + } + } + + public boolean isAsync() { + return async; + } + + public void setAsync(final boolean async) { + this.async = async; + } + + public String getBatchNumMessages() { + return batchNumMessages; + } + + public void setBatchNumMessages(final String batchNumMessages) { + this.batchNumMessages = batchNumMessages; + } + + @Override + public boolean equals(final Object obj){ + return EqualsBuilder.reflectionEquals(this, obj); + } + + @Override + public int hashCode() { + return HashCodeBuilder.reflectionHashCode(this); + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ZookeeperConnect.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ZookeeperConnect.java new file mode 100644 index 0000000000..aa31a4d3fc --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ZookeeperConnect.java @@ -0,0 +1,60 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import org.springframework.integration.kafka.core.ZookeeperConnectDefaults; + +/** + * @author Soby Chacko + */ +public class ZookeeperConnect { + private String zkConnect = ZookeeperConnectDefaults.ZK_CONNECT; + private String zkConnectionTimeout = ZookeeperConnectDefaults.ZK_CONNECTION_TIMEOUT; + private String zkSessionTimeout = ZookeeperConnectDefaults.ZK_SESSION_TIMEOUT; + private String zkSyncTime = ZookeeperConnectDefaults.ZK_SYNC_TIME; + + public String getZkConnect() { + return zkConnect; + } + + public void setZkConnect(final String zkConnect) { + this.zkConnect = zkConnect; + } + + public String getZkConnectionTimeout() { + return zkConnectionTimeout; + } + + public void setZkConnectionTimeout(final String zkConnectionTimeout) { + this.zkConnectionTimeout = zkConnectionTimeout; + } + + public String getZkSessionTimeout() { + return zkSessionTimeout; + } + + public void setZkSessionTimeout(final String zkSessionTimeout) { + this.zkSessionTimeout = zkSessionTimeout; + } + + public String getZkSyncTime() { + return zkSyncTime; + } + + public void setZkSyncTime(final String zkSyncTime) { + this.zkSyncTime = zkSyncTime; + } +} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/package-info.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/package-info.java new file mode 100644 index 0000000000..0ec591dea7 --- /dev/null +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/package-info.java @@ -0,0 +1,4 @@ +/** + * Provides various support classes used across Spring Integration Kafka Components. + */ +package org.springframework.integration.kafka.support; diff --git a/spring-integration-kafka/src/main/resources/META-INF/spring.handlers b/spring-integration-kafka/src/main/resources/META-INF/spring.handlers new file mode 100644 index 0000000000..1bee2b0d06 --- /dev/null +++ b/spring-integration-kafka/src/main/resources/META-INF/spring.handlers @@ -0,0 +1 @@ +http\://www.springframework.org/schema/integration/kafka=org.springframework.integration.kafka.config.xml.KafkaNamespaceHandler diff --git a/spring-integration-kafka/src/main/resources/META-INF/spring.schemas b/spring-integration-kafka/src/main/resources/META-INF/spring.schemas new file mode 100644 index 0000000000..b5240f8442 --- /dev/null +++ b/spring-integration-kafka/src/main/resources/META-INF/spring.schemas @@ -0,0 +1,2 @@ +http\://www.springframework.org/schema/integration/kafka/spring-integration-kafka-1.0.xsd=org/springframework/integration/config/xml/spring-integration-kafka-1.0.xsd +http\://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd=org/springframework/integration/config/xml/spring-integration-kafka-1.0.xsd diff --git a/spring-integration-kafka/src/main/resources/META-INF/spring.tooling b/spring-integration-kafka/src/main/resources/META-INF/spring.tooling new file mode 100644 index 0000000000..d4079816e8 --- /dev/null +++ b/spring-integration-kafka/src/main/resources/META-INF/spring.tooling @@ -0,0 +1,4 @@ +# Tooling related information for the integration Kafka namespace +http\://www.springframework.org/schema/integration/kafka@name=integration Kafka Namespace +http\://www.springframework.org/schema/integration/kafka@prefix=int-kafka +http\://www.springframework.org/schema/integration/kafka@icon=org/springframework/integration/config/xml/spring-integration-kafka.gif diff --git a/spring-integration-kafka/src/main/resources/org/springframework/integration/config/xml/spring-integration-kafka-1.0.xsd b/spring-integration-kafka/src/main/resources/org/springframework/integration/config/xml/spring-integration-kafka-1.0.xsd new file mode 100644 index 0000000000..1c716fe086 --- /dev/null +++ b/spring-integration-kafka/src/main/resources/org/springframework/integration/config/xml/spring-integration-kafka-1.0.xsd @@ -0,0 +1,451 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The topic configured by this configuration. + + + + + + + + + + + + + + + + + Custom implemenation of a Kafka Encoder for encoding message values. + + + + + + + Custom implemenation of a Kafka Encoder for encoding message keys. + + + + + + + Class type used for the key + + + + + + + Class type used for the value + + + + + + + + + + + + + + + + + Custom Kafka key partitioner. + + + + + + + Indicates if this producer is async or not. + + + + + + + number of messages to batch at this producer. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Kafka Server Bean Name + + + + + + + Kafka Server Bean Name + + + + + + + + + + + + + + + + + + + + + + + + + Kafka Server Bean Name + + + + + + + + + + The definition for the Spring Integration Kafka + Inbound Channel Adapter. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Kafka Server Bean Name + + + + + + + + + + + Identifies the underlying Spring bean definition, which is an + instance of either 'EventDrivenConsumer' or 'PollingConsumer', + depending on whether the component's input channel is a + 'SubscribableChannel' or 'PollableChannel'. + + + + + + + Flag to indicate that the component should start automatically + on startup (default true). + + + + + + + + + + + + Defines kafka outbound channel adapter that writes the contents of the + Message to kafka broker. + + + + + + + + + + + Identifies the underlying Spring bean definition, which is an + instance of either 'EventDrivenConsumer' or 'PollingConsumer', + depending on whether the component's input channel is a + 'SubscribableChannel' or 'PollableChannel'. + + + + + + + Flag to indicate that the component should start automatically + on startup (default true). + + + + + + + + + + Kafka producer context reference. + + + + + + + + + + + + + + + diff --git a/spring-integration-kafka/src/main/resources/org/springframework/integration/config/xml/spring-integration-kafka.gif b/spring-integration-kafka/src/main/resources/org/springframework/integration/config/xml/spring-integration-kafka.gif new file mode 100644 index 0000000000000000000000000000000000000000..41b369fece0e576ecd920400b221fd33abbaf5c4 GIT binary patch literal 572 zcmZ?wbhEHb6krfwc*Xz%|NsBj^POrMHnn5wI!WKA$M#t>l?H4BPzO?b&{cRWSFFSB< z^|`x}P0QyVy|U`SiF40CoO%A?(2mZdyE;$q?mn}(`^J&Ji~DRU^KH0`_S=*f zu`(xWd1~172%niTfzv{QrUeE70}32q)Fc#tvM@3*q%i1!3doJ z$QZroAYKk;o)!}g_c$BZP}??kHkSD;%JZ2d6u9}BT8+&ujM`ePV%-d#6T`zi);F^_ zIR!9@O3I6^oV(z-tdN`(pX{#l&0dZgror~-%^VCYeSN$`0@iCL`334n#7Fs`w{uBO x(B)aWd{Qfm5=#n`xT2uIN}aG328D}QI)&6(d>(9QbkfS;6w=UeP!nLV1^^EL*)#wE literal 0 HcmV?d00001 diff --git a/spring-integration-kafka/src/reference/docbook/SIAdapterLowerPrefix.xml b/spring-integration-kafka/src/reference/docbook/SIAdapterLowerPrefix.xml new file mode 100644 index 0000000000..2ff42a43f1 --- /dev/null +++ b/spring-integration-kafka/src/reference/docbook/SIAdapterLowerPrefix.xml @@ -0,0 +1,72 @@ + + + Kafka Adapter + + The Spring Integration Kafka Adapter provides... + + + + Outbound Channel adapter + + + Outbound Gateway + + + Inbound Channel Adapter + + + +
+ Java Implementation + Each of the provided components will use the + org.springframework.integration.kafka.core.KafkaExecutor + class... + +
+
+ Common Configuration Attributes + + Certain configuration parameters are shared amongst all Kafka + components and are described below: + + + auto-startup + + Lifecycle attribute signaling if this component should + be started during Application Context startup. + Defaults to true. + Optional. + + + id + + Identifies the underlying Spring bean definition, which + is an instance of either EventDrivenConsumer + or PollingConsumer. + Optional. + + +
+ +
+ Outbound Channel Adapter + + The Kafka Outbound channel adapter allows you to... + +
+
+ Outbound Gateway + + Outbound gateways are similar to outbound channel adapters except that it can also be used to + get a result on the reply channel after performing + the given... + +
+
+ Inbound Channel Adapter + + An inbound channel adapter is used to execute... + +
+ + diff --git a/spring-integration-kafka/src/reference/docbook/history.xml b/spring-integration-kafka/src/reference/docbook/history.xml new file mode 100644 index 0000000000..71b4f74b66 --- /dev/null +++ b/spring-integration-kafka/src/reference/docbook/history.xml @@ -0,0 +1,4 @@ + + + Change History + diff --git a/spring-integration-kafka/src/reference/docbook/images/logo.png b/spring-integration-kafka/src/reference/docbook/images/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..e1c2a0e3c7864b4c2495fa75fed9733af2558ff9 GIT binary patch literal 9627 zcmV;MC1l!(P)004R=004l4008;_004mL004C`008P>0026e000+nl3&F} z001Q+Nkla zCi#~HZyJIZTG9$KUioZd`!}Or{bFjSBIkwcKiM>eFRp0xnmJn*9lLahrv&$&JY*67 z(+68b0=h~6HOW6Ax03H2xpI^z2mp-7iNjZqC1fNu+21DlheV}SfldP$0RVvGj83D` zYn$+AUU*(tq&lBlvd6#{HbJLRBr3Q=M7thPUIL6BRHe~vdxZOizF0p2YxMY+VVRtw z{J)a#lxtKTPHuxb_A@@`c+JC{*?A>UhwVMG&v8YnL)_!_GJc*Os9;lPd)Kb(oX2nMYQd4K`Wf?m5|;z!G; ze2igOjZf*AoLf&~Sgsra>*+M1rTgM02>#EC?fUQN%~LCOuelO`6<7lZ#UPSCLnRB0 zVZo%2R!sTC><1OfKUlsT#{~dXPVp`b@D}bLV%R^&L?VFPqP#+7adCMG!x#X=0D!=7 z2dTZCSmx={Oe_@DW9fK=u;(mGBp@kkd!=U#LP}8oJ1oay7*620C7P?r?}qX!HFga9Wg zJ4Z*Uy^pJxpWJU)K-+_7)B*3q(xm05>e zO}M%8_w|ZWg}u~%Zp@7S5xr}ymM3{>hp(NuoN(i5i2~!;u5G(58NWb42>{^O%~Kop zZz|VmUF@A_jd-hHyPlO@-LwAqm811{6(K|tgomSh8~@NDQT@Ywf~~@$fn}~HT;G4? z*n=kz^UDg$wW>m85r)-As=i@T%?*rd~tH;*t-EcMW7OzzUARs6p2*3%bI@~-5 zJP#}bJO>2Tu2s9O^EP%2Y*)3Nu`Ax#eegS=h4nf$06Y&E1_Ij0!*}va<7N(d-Oa(p z?7KzfC0*Yem=u2lXd&qJUM*T*TXWLOrJ3=6&)mK6^?@xX;?5h?N?;klZ~z31;hTlq zXAXTmtspb{-TwJcl7JS1PVXPw{_2{O?hdZT&yHL9=C22~KfkL80nY1>0QDiNU5oHLn=aX+|H`$hd7D4{?)dir z0AiUr>ct?J`+WWOmuA72Q}Rnj;bsD-v6`eKj$wjy#RieU+s$rJqa#nHRnPzS4t+f_^HF}z6R>v#3my-XvTAvC4diH4}l93Zv&)od`I>1Sw#g*EQW5J*Yod$Fo&})Fv z1JBjy6VTGs%tyIJ1qz7nFd!+A*d?UgdwI$Dul9XoZqyK9)r^5iC``;u?lXVbu46k` zjt7wxaJ;hT2mzz_be02v-k=ATG5r^YgI1%~scr5nU^s}Sde*RF=h8V_7I8HO+v5BoZACZhst4M48Z8cj&82@E{;+M9LG40 zFDx(0Da->^DNwY@j$i-=t-P6kGbgutwFVD*of)G>kS1+v6k4oj3?FP-KoZ2<5z}i- zJ608dlprxHdEVB=$>|S4Vqax#^T08{7yuwRH1e%MlK=q2uxitfVfA@$hzEu-Z-Ox# z2m*|L8$p6d_Ra4bLYfE89P)atsnkzhJL}^0lOVOP7R7KZum<2b;Ca)+tMJWeG1RR} zpO3%9tmNUb<5JQdn76qV1BBz#22GNfiySkGHfi*aNkaA<#f;O%sw>&WJH4G>znwNp z&*uBPzTc40>}hG?nuD7GP~{})G_6{+o;7S*WUKZbj&9C&PB?~hJomIzk?}O^{*wnM zZk@Z85?87!Wm#5c=e&H%CvseD=%gp`8YJyLeV@=Cj?aPhL0x z61z(9vIgMT0N;>)?R&QJYGL$T)jCafVJ?p0EXQs+{(bJ#EFef*@RwbfJ2G}$%H#X> zf)9Y_?HpW(bbQH-ub_>4T6UbkE%HK0T%29>y_?-2U)R|WvSx6cvZco-7)IHKzXur! zY3WH&u_O$-F712nUHrX=qkD}v{XBg9JbZeFcbz?Ada zw~ObFc^k~y{l8M;HXPjc+n;-Yc{Z;kr7bUAuw!X-NTigO)Mc`ZN}d8im^7AF?d0M) zb>PG){l;0rgq0nV~sS;ae(^OqsS*f!>mbJKxiz-ZkN zk73w?(Q|8quZB2D9s0KGIlB9Z=Ya1Se_{4HMN-khk-Is?eF2Rs)$W%2Kee+cq!(O8|S9|~PVqsH|a16p;3 zC;A{gsi?fjL|+u#@5+t=|+VYMq+ty3d}8b z#Wbj6-&v#HsZ@caATcZDa>9-JX3e1cWYVG~6Bc6sY*)L6L;@)=sX&4(EXpf?2FJ?@ zH;aq%E7=pRdTifUW1;HJa2|!wg%}c-6K@*nyX6%Dz*2@G;tmd zF$i&h5$TGJW!l@;wiD(PM2KaT5ERmD_imiA;l0${w10Tl)%&B~c6Rdyjx&1qyjpqg z?!`La)@z6M92q-q{y)D|TrDHijxulBtN4r%PW>Se>Q0` zj{gg#>CQ4IT0mFgtcvN2&q%7INIA~KwOMpXWW&GFOjHt|p0N4o4(mv5f{af~U|5E# zo_A+?3IMVRe}Xb?oPjNLkVGoVP7pY))WqeL{O;vE+44rjLea8`3x_|xuU3_s))G#D zNK#f_a_IcNLuY@Z?3`TfotwEd3-bx?(6UW*aAb2gFK0Wazhi161KaOFJOBc~fFSdW z3mA^A9gPBO=ois5-0$Bh&aKN`(;4)os!~%_4gfso@8K&EivJe;Z9cj~TdH_|%v&9Q z37N^c#rZyN-qk&@W>z|X73&V?cvWHfIag`lr)9?gFbpHIOMY;(A0i;Fh#~s4?eX)% z?QecHL!rn4T4EYOoG@mk7=u2yATu{N?biJ(-valsEDtF8zrHzDPY5Qkxu){tFTe%ji0hIbx#`m1BZ z`-~<@5;RIvd4bWy5Cl*{pv9&?l1O+QzkS~~F|lL2y*Kc5+=a%RLY_AlY1LLiyxh?P z{(DkA5r7m>CgRwtf`0Dtq(R01!z7!<@y%sd(!04e`c!YsK2ogIE z!R0&W2Y)!?(_dEoJ$4S!0-CCWlI0QqUjrWkR4rn|@C-$Eim@mH&RVE%TZ_B^k*b(%PhAx%by8G{%cqg1+Nwf>Q9two1K zZg_iD!nSLFe6eroo1ez?858Img5e}+RG`yoMl#d*#D|f1SP5#mjGKaR*mTYrR1gENV;uG7BPlyVVx@IA4@`DPN!bI z|LeYOyI1IH922G_$q&1il4?`pydmG0RZWeV*A!pk&DE51cQpZV*|nd)wn zt6hJQ(2ut%hlXKriG0VkW`RYOL(Cqz{969{>AX8Wkq>K`HgIU?>|Xk zITkPscn*`;`Fr@L3dfkdW>#Hhfig1)Wyn>LyrYM+T*kTMLNK$50{uoB8 z^eG6jjXu6OhIa~$`gZR6o$r245roOsU>NZHU&(O@8pKzp6=YsXxcLl!JSnk@4vl)T z@YmWe7%D*Pk~14@%GQZX_)+g=;v@>vg-%?RmcANKynrSc*TLUi&P}K@>p> zDWPqBU9*3)zDm?0AYDA2CIIl(2ErJIbDWCd%Y~F%MIMDmdVs$%d#2H8O4Z8BwGN0P zDf1*0LFN|b#HT0L-{7s3yW0+J15#kp={n78y+_k8)G1W};N#}qwncc=LY~8y4*ghv zArC-m?vwn&{CZ3%UW3M~8gttyLeZ@U*A8DlS@l^nhcQw*#Ck6fLx=+e0>ZgU9@#pt zH@h(Rz~#diA6zX}mp%7ohGUog^3~&vM4+A}a#+U4&5Jb4m5@|XiDL9;lkPmeUsr>V zvQsConqF33V!|Jzw|Dg%-F*bSXhI8wW4et1j;lllhD}>HZ~eioN{zDi`v#WTclk*F zk49zYrvX7(6#rHR*cfKu7_C8TOZhQ|^E~(E-i?DJU#i$hi{rGX<7-K|3jq_@#y^2g zDeHmeWHBE&JuGjK>ea&IthBO}ceA{_0w(gQLVw?jjHK3U-j#Bc2Bg z%PTIpl@ga%oR75XYTKC#+5o?>fgStSs~IH4%HnswnZJ7fH=Tnb!+e89v?wt%`R1eB z3F!&IvL^5WfL7Bou-*KZXTggpt`07rjC+6Dm(wc?qX-fR^7g} zt-ZbT&Ue3dcXWdnR$lKv;l`soJN9n^v8=MTS11BN!1H>&Mz2!?h5;pjioIs2a{mg3 zVfNLxuvPQ_+O=pyk(6mx5T99?OS8T6Gn>`Z*RDJ#9oM03bx*M4)HOI`J3TsvRv5nr4fn zlCx6-y#m1IbRm*FUQv~j6{|d1tQ=Dq^r)5&2mbUCyID`k)_N9j$un) zTM!Y^0~iBn%7J46p?cj%1PFmn3I=^-c$YuEJo?ViH|j@Tkays`(R09{1GWxIh+{#m zZ0Q&Jd+g8SdX6-Up=zZGIBxl2Rsu=@CVc_G^B|VWYHXIMz#QVS`gdnm0vJ{0&1<7p zLW_U=VDpN1R=GR7gIWm&9UvZzE1}|#Wk6LX6-j4}dPho&fj5yV1dfOK2Gxnb(B?sH z{ab^vhJd0jKfK}Y=q7P;2iuZ;6m>i0-lYfEsy=g(^|2GTMZARs9-;`xa(x2Jl>y*9 zVuY03s)H-r1%zK%ef;N-e;hw>qPw#kcn%B((5gYL1fANn|6FJMrv?~?Ajp1E1Ah2u z=jGKWyN7jpo`L6RO1S2&&o;lmN$%XtRMVhUgH8il6==#quXVC_o;~jUKUW^>eI@?Ul-&Z=;F~FlqI0kKzXT$tEZ1^W&K-3%eG+9gH{S! zHRzen!5zuxA=}s8TTF{gqfuV!?(XAcuFi!bs0E|vTwQ;5$+XX-T1V3Yk!k57WK?qg z-YsXm{LYz`hhm}!mnch2>JTB3fbJ3!Rj2>wV(uo*{S&@1%=8Iqi&`M(pl!%qM3h0%Ql^ZBYoVxDT{xsv4!b_ zrwr>n=-7=vj^8?6P*kAjScyRB>FU)rBx-2q0nJ^#>S@8#S^o9Rm238FP#JVmf#B^C zGyE(*!!$tdyK%;sn-6YLF$SrSe(#kz&D|=$>#c#4vK86qZvP1w9^x0WVDy~lHYj)U z*f3+orbFMV^%|*2`rga4JnF9V^OyU^j-NmO^>?@xk-7c%hta8_1qEDsxr7WiD_?F&k@l>OePrdr{m7|4DSL0k&SdTNu?KLl&DIp zc`c-~y*R?MbR0s6F@g^PD-w9R%KA4}C`S?`EfCTIq2=JS%<05!03QG}WWMY9EKLcm z1z!U@Qz0MjT&5~jn83#x>|MS4NA#|dJ}{FQnHMv|!Nn6e7KEZ32{&<^=oa1u7@bYC zK`6?KYF`Q<6NTZJz%nKod*%uowU~a(6=S(bW{0W@4yr_gn29cn-++j@9y#Duv z;n;;cmM;4C6Cf#5Mi=pfNbciJh^jk1|ao2? z*Eq?uKKuzt!N!>@<#JzO+4$6Z_tO&Qj+kDZZ?teN=!2WLs=cET1Kjd50*=?X$ok10 zUT&J;|E!d%$`^; zF5hze`$>Jr`1*%g2SJ1Ypi@wX+Jq|0mm{!2wE~RA9IXb!D+;Fyq z+^<%AR2r8)PGd;J>#hYvh(*8$xXt%=p7LKC?!A|f{B~i#L8s{y+G*a%882KVbE8X^ zBB%Gl;i*ZtK_meLz%f8lUEB8feDdP%VV&#$tflua+)qpRFB{zb}#t0cEJV9mBnR@ zQ|lksbrw0t>nK9j(j&I%cEbM!vE}|ZMOjh5g(L1I-UN}X7Kev{rmSm3pTnQ-HhV-! zYQgfXq8$V-GN(Qf$El>Ekd_{?4&siD{!^249w%p|6swgqCGcE;n`$$&J2BYxYQoLoOU6Sl5p*Lj+`nRNBezoS_4!~Z z^q`;vKm^c3+$VkYwxX=i+IDgZb4MtDNo*57@w zY0PJnmCEAE3MM*DM0l4U=WlK9W`-^)EdMh(e+fc-3MLOBmP1;pFvQRGZCAU2e}|Hf z-L-PZ?u`K819J|YmN^Xc=IePPyeT+Bs7Q+B1=q>K}POlhj8cVUyM&Q zI_>N0=0G%bnAL}2S*u!Gy6a2m4DW7RT=8V%p>5C^R{p-h+Kz?K=my;>Y=F~O`@SyI zl-n>wlUKhAonY$PSw%|IL?mV=pZM#H&Y-iZozr*DLr)?+4eqQ*Uz047G8sNBI?Rz-M%s4J(+B*rkAF#aANVp*2d8Fg`auMb2 z=lX72-(5Z~(6s4veVugcQnCO6?s2w*x@5z|%?E!%5QFNzxUwX(PIA z;^`tk5WDL}((S)e;_jy2KY9Dyv2%N_CjND4^@&hx?scI^0gzCvs(Fq=p|S`e9=CF@ zzH!@8w$s-Qh;L^%X19NQe=Z5K@tQYzPYQHmrJu}hmj(ZAh8gxT|oHX zf74+p33_aJ>2T*=V=k zi-n@K)4%B2y0fA%&->Z+$p9c0_=*L-@UO}rx6keTBf)_h!5;6jybq+Q9ACT!A--Y%7eI(dGXMCUw^pQd|^jkDi zQED~0m6G!$Y+4uSf4CNpbLIG^`5m&9l8F8!?0%1 z1%sCy@PRIPZ}Ipqr`7c`v1G}T|My9r&cTr_0)klpsetxyaB1JF{l}9RFPX5=q3R{J zmWh+haZHaGXBWBJpmh;RJBCFsdt=#xS7zC;$A`d)0TF!yd|L4sArXq?P9Ab6ce%ZD zpALQ7_*-|t@VzGwWfIxMzGDTHjb1fHkVB$g^7jc+>a`BEL~ifWrET}G=WU2;6@goM zgF&EI!0@iaL{d9F%eXr@d)T>j4(rsj4zq4c{;f%xq|r=&(-1VtzbOAV0DzS)V@r=* R@&Et;07*qoM6N<$f(p*0Lx literal 0 HcmV?d00001 diff --git a/spring-integration-kafka/src/reference/docbook/index.xml b/spring-integration-kafka/src/reference/docbook/index.xml new file mode 100644 index 0000000000..bdb072fffc --- /dev/null +++ b/spring-integration-kafka/src/reference/docbook/index.xml @@ -0,0 +1,67 @@ + + + + Spring Integration Kafka Adapter + Kafka Adapter ${version} + Spring Integration + ${version} + + + + + + + + + + + + + + Soby Chacko + + + © SpringSource Inc., 2012 + + + + + + + What's new? + + + For those who are already familiar with Spring Integration, this + chapter + provides a brief overview of the new features of version 2.2. If you are + interested in the changes and features, that were introduced in + earlier + versions, please take a look at chapter: + + + + + + + + + + Integration Adapters + + This section covers the various Channel Adapters and Messaging + Gateways provided + by Spring Integration to support Message-based communication with + external systems. + + + + + + Appendices + + Advanced Topics and Additional Resources + + + + diff --git a/spring-integration-kafka/src/reference/docbook/resources.xml b/spring-integration-kafka/src/reference/docbook/resources.xml new file mode 100644 index 0000000000..456ebba6f6 --- /dev/null +++ b/spring-integration-kafka/src/reference/docbook/resources.xml @@ -0,0 +1,15 @@ + + + Additional Resources + +
+ Spring Integration Home + + The definitive source of information about Spring Integration is the + Spring Integration Home at + http://www.springsource.org. That site serves as a hub of + information and is the best place to find up-to-date announcements about the project as well as links to + articles, blogs, and new sample applications. + +
+ diff --git a/spring-integration-kafka/src/reference/docbook/whats-new.xml b/spring-integration-kafka/src/reference/docbook/whats-new.xml new file mode 100644 index 0000000000..ddf2480cbc --- /dev/null +++ b/spring-integration-kafka/src/reference/docbook/whats-new.xml @@ -0,0 +1,8 @@ + + + What's new? + + This chapter provides an overview of the new features and improvements + that have been added to the Kafka Adapter: + + diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests.java new file mode 100644 index 0000000000..50ae6fccb9 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests.java @@ -0,0 +1,47 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.config.xml; + +import junit.framework.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.integration.kafka.support.ConsumerMetadata; +import org.springframework.integration.kafka.support.KafkaConsumerContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +/** + * @author Soby Chacko + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration +public class KafkaConsumerContextParserTests { + + @Autowired + private ApplicationContext appContext; + + @Test + @SuppressWarnings("unchecked") + public void testConsumerContextConfiguration() { + final KafkaConsumerContext consumerContext = appContext.getBean("consumerContext", KafkaConsumerContext.class); + Assert.assertNotNull(consumerContext); + + final ConsumerMetadata cm = appContext.getBean("consumerMetadata_default1", ConsumerMetadata.class); + Assert.assertNotNull(cm); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests.java new file mode 100644 index 0000000000..e7525b9bb7 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests.java @@ -0,0 +1,45 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.config.xml; + +import junit.framework.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.integration.endpoint.SourcePollingChannelAdapter; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration +public class KafkaInboundAdapterParserTests { + + @Autowired + private ApplicationContext appContext; + + /** + * Test method for {@link org.springframework.integration.kafka.config.xml.KafkaInboundChannelAdapterParser#parseSource(org.w3c.dom.Element, org.springframework.beans.factory.xml.ParserContext)}. + */ + @Test + public void testParseSourceElementParserContext() throws Exception { + final SourcePollingChannelAdapter adapter = appContext.getBean("kafkaInboundChannelAdapter", + SourcePollingChannelAdapter.class); + + Assert.assertNotNull(adapter); + Assert.assertFalse(adapter.isAutoStartup()); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests.java new file mode 100644 index 0000000000..3c69fcf7d6 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests.java @@ -0,0 +1,46 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.config.xml; + +import junit.framework.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.integration.endpoint.PollingConsumer; +import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler; +import org.springframework.integration.kafka.support.KafkaProducerContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration +public class KafkaOutboundAdapterParserTests { + + @Autowired + private ApplicationContext appContext; + + @Test + public void testOutboundAdapterConfiguration(){ + final PollingConsumer pollingConsumer = appContext.getBean("kafkaOutboundChannelAdapter", PollingConsumer.class); + final KafkaProducerMessageHandler messageHandler = appContext.getBean(KafkaProducerMessageHandler.class); + Assert.assertNotNull(pollingConsumer); + Assert.assertNotNull(messageHandler); + final KafkaProducerContext producerContext = messageHandler.getKafkaProducerContext(); + Assert.assertNotNull(producerContext); + Assert.assertEquals(producerContext.getTopicsConfiguration().size(), 2); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests.java new file mode 100644 index 0000000000..ddf9718e77 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests.java @@ -0,0 +1,73 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.config.xml; + +import junit.framework.Assert; +import kafka.javaapi.producer.Producer; +import kafka.serializer.Encoder; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.integration.kafka.support.KafkaProducerContext; +import org.springframework.integration.kafka.support.ProducerConfiguration; +import org.springframework.integration.kafka.support.ProducerMetadata; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.util.Map; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration +public class KafkaProducerContextParserTests { + + @Autowired + private ApplicationContext appContext; + + @Test + @SuppressWarnings("unchecked") + public void testProducerContextConfiguration(){ + final KafkaProducerContext producerContext = appContext.getBean("producerContext", KafkaProducerContext.class); + Assert.assertNotNull(producerContext); + + final Map topicConfigurations = producerContext.getTopicsConfiguration(); + Assert.assertEquals(topicConfigurations.size(), 2); + + final ProducerConfiguration producerConfigurationTest1 = topicConfigurations.get("producerConfiguration_test1"); + Assert.assertNotNull(producerConfigurationTest1); + final ProducerMetadata producerMetadataTest1 = producerConfigurationTest1.getProducerMetadata(); + Assert.assertEquals(producerMetadataTest1.getTopic(), "test1"); + Assert.assertEquals(producerMetadataTest1.getCompressionCodec(), "0"); + Assert.assertEquals(producerMetadataTest1.getKeyClassType(), java.lang.String.class); + Assert.assertEquals(producerMetadataTest1.getValueClassType(), java.lang.String.class); + + final Encoder valueEncoder = appContext.getBean("valueEncoder", Encoder.class); + Assert.assertEquals(producerMetadataTest1.getValueEncoder(), valueEncoder); + Assert.assertEquals(producerMetadataTest1.getKeyEncoder(), valueEncoder); + + final Producer producerTest1 = appContext.getBean("prodFactory_test1", Producer.class); + Assert.assertEquals(producerConfigurationTest1, new ProducerConfiguration(producerMetadataTest1, producerTest1)); + + final ProducerConfiguration producerConfigurationTest2 = topicConfigurations.get("producerConfiguration_" + "test2"); + Assert.assertNotNull(producerConfigurationTest2); + final ProducerMetadata producerMetadataTest2 = producerConfigurationTest2.getProducerMetadata(); + Assert.assertEquals(producerMetadataTest2.getTopic(), "test2"); + Assert.assertEquals(producerMetadataTest2.getCompressionCodec(), "0"); + + final Producer producerTest2 = appContext.getBean("prodFactory_test2", Producer.class); + Assert.assertEquals(producerConfigurationTest2, new ProducerConfiguration(producerMetadataTest2, producerTest2)); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests.java new file mode 100644 index 0000000000..6acfa87e94 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests.java @@ -0,0 +1,59 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.config.xml; + +import junit.framework.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.integration.kafka.core.ZookeeperConnectDefaults; +import org.springframework.integration.kafka.support.ZookeeperConnect; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +/** + * @author Soby Chacko + * @since 1.0 + * + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration +public class ZookeeperConnectParserTests { + + @Autowired + private ApplicationContext appContext; + + @Test + public void testCustomKafkaBrokerConfiguration() { + final ZookeeperConnect broker = appContext.getBean("zookeeperConnect", ZookeeperConnect.class); + + Assert.assertEquals("localhost:2181", broker.getZkConnect()); + Assert.assertEquals("10000", broker.getZkConnectionTimeout()); + Assert.assertEquals("10000", broker.getZkSessionTimeout()); + Assert.assertEquals("200", broker.getZkSyncTime()); + } + + @Test + public void testDefaultKafkaBrokerConfiguration() { + final ZookeeperConnect broker = appContext.getBean("defaultZookeeperConnect", ZookeeperConnect.class); + + Assert.assertEquals(ZookeeperConnectDefaults.ZK_CONNECT, broker.getZkConnect()); + Assert.assertEquals(ZookeeperConnectDefaults.ZK_CONNECTION_TIMEOUT, broker.getZkConnectionTimeout()); + Assert.assertEquals(ZookeeperConnectDefaults.ZK_SESSION_TIMEOUT, broker.getZkSessionTimeout()); + Assert.assertEquals(ZookeeperConnectDefaults.ZK_SYNC_TIME, broker.getZkSyncTime()); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/serializer/AvroBackedKafkaSerializerTest.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/serializer/AvroBackedKafkaSerializerTest.java new file mode 100644 index 0000000000..a12e9085be --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/serializer/AvroBackedKafkaSerializerTest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.serializer; + +import junit.framework.Assert; +import org.junit.Test; +import org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaDecoder; +import org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaEncoder; +import org.springframework.integration.kafka.test.utils.TestObject; + +/** + * @author Soby Chacko + * @since 1.0 + */ +public class AvroBackedKafkaSerializerTest { + @Test + @SuppressWarnings("unchecked") + public void testDecodePlainSchema() { + final AvroBackedKafkaEncoder avroBackedKafkaEncoder = new AvroBackedKafkaEncoder(TestObject.class); + + final TestObject testObject = new TestObject(); + testObject.setTestData1("\"Test Data1\""); + testObject.setTestData2(1); + + final byte[] data = avroBackedKafkaEncoder.toBytes(testObject); + + final AvroBackedKafkaDecoder avroBackedKafkaDecoder = new AvroBackedKafkaDecoder(TestObject.class); + final TestObject decodedFbu = (TestObject) avroBackedKafkaDecoder.fromBytes(data); + + Assert.assertEquals(testObject.getTestData1(), decodedFbu.getTestData1()); + Assert.assertEquals(testObject.getTestData2(), decodedFbu.getTestData2()); + } + + @Test + @SuppressWarnings("unchecked") + public void anotherTest() { + final AvroBackedKafkaEncoder avroBackedKafkaEncoder = new AvroBackedKafkaEncoder(java.lang.String.class); + final String testString = "Testing Avro"; + final byte[] data = avroBackedKafkaEncoder.toBytes(testString); + + final AvroBackedKafkaDecoder avroBackedKafkaDecoder = new AvroBackedKafkaDecoder(java.lang.String.class); + final String decodedS = (String) avroBackedKafkaDecoder.fromBytes(data); + + Assert.assertEquals(testString, decodedS); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ConsumerConfigurationTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ConsumerConfigurationTests.java new file mode 100644 index 0000000000..f41f18f283 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ConsumerConfigurationTests.java @@ -0,0 +1,306 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import junit.framework.Assert; +import kafka.consumer.ConsumerIterator; +import kafka.consumer.KafkaStream; +import kafka.javaapi.consumer.ConsumerConnector; +import kafka.message.MessageAndMetadata; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author Soby Chacko + */ +public class ConsumerConfigurationTests { + @Test + @SuppressWarnings("unchecked") + public void testReceiveMessageForSingleTopicFromSingleStream() { + final ConsumerMetadata consumerMetadata = Mockito.mock(ConsumerMetadata.class); + final ConsumerConnectionProvider consumerConnectionProvider = + Mockito.mock(ConsumerConnectionProvider.class); + final MessageLeftOverTracker messageLeftOverTracker = Mockito.mock(MessageLeftOverTracker.class); + final ConsumerConnector consumerConnector = Mockito.mock(ConsumerConnector.class); + + Mockito.when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector); + + final ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(consumerMetadata, + consumerConnectionProvider, messageLeftOverTracker); + consumerConfiguration.setMaxMessages(1); + + final KafkaStream stream = Mockito.mock(KafkaStream.class); + final List> streams = new ArrayList>(); + streams.add(stream); + final Map>> messageStreams = new HashMap>>(); + messageStreams.put("topic", streams); + + Mockito.when(consumerConfiguration.getConsumerMapWithMessageStreams()).thenReturn(messageStreams); + final ConsumerIterator iterator = Mockito.mock(ConsumerIterator.class); + Mockito.when(stream.iterator()).thenReturn(iterator); + final MessageAndMetadata messageAndMetadata = Mockito.mock(MessageAndMetadata.class); + Mockito.when(iterator.next()).thenReturn(messageAndMetadata); + Mockito.when(messageAndMetadata.message()).thenReturn("got message"); + Mockito.when(messageAndMetadata.topic()).thenReturn("topic"); + Mockito.when(messageAndMetadata.partition()).thenReturn(1); + + final Map>> messages = consumerConfiguration.receive(); + Assert.assertEquals(messages.size(), 1); + Assert.assertEquals(messages.get("topic").size(), 1); + Assert.assertEquals(messages.get("topic").get(1).get(0), "got message"); + + Mockito.verify(stream, Mockito.times(1)).iterator(); + Mockito.verify(iterator, Mockito.times(1)).next(); + Mockito.verify(messageAndMetadata, Mockito.times(1)).message(); + Mockito.verify(messageAndMetadata, Mockito.times(1)).topic(); + } + + @Test + @SuppressWarnings("unchecked") + public void testReceiveMessageForSingleTopicFromMultipleStreams() { + final ConsumerMetadata consumerMetadata = Mockito.mock(ConsumerMetadata.class); + final ConsumerConnectionProvider consumerConnectionProvider = + Mockito.mock(ConsumerConnectionProvider.class); + final MessageLeftOverTracker messageLeftOverTracker = Mockito.mock(MessageLeftOverTracker.class); + + final ConsumerConnector consumerConnector = Mockito.mock(ConsumerConnector.class); + + Mockito.when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector); + + final ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(consumerMetadata, + consumerConnectionProvider, messageLeftOverTracker); + consumerConfiguration.setMaxMessages(3); + + final KafkaStream stream1 = Mockito.mock(KafkaStream.class); + final KafkaStream stream2 = Mockito.mock(KafkaStream.class); + final KafkaStream stream3 = Mockito.mock(KafkaStream.class); + final List> streams = new ArrayList>(); + streams.add(stream1); + streams.add(stream2); + streams.add(stream3); + final Map>> messageStreams = new HashMap>>(); + messageStreams.put("topic", streams); + + Mockito.when(consumerConfiguration.getConsumerMapWithMessageStreams()).thenReturn(messageStreams); + final ConsumerIterator iterator1 = Mockito.mock(ConsumerIterator.class); + final ConsumerIterator iterator2 = Mockito.mock(ConsumerIterator.class); + final ConsumerIterator iterator3 = Mockito.mock(ConsumerIterator.class); + + Mockito.when(stream1.iterator()).thenReturn(iterator1); + Mockito.when(stream2.iterator()).thenReturn(iterator2); + Mockito.when(stream3.iterator()).thenReturn(iterator3); + final MessageAndMetadata messageAndMetadata1 = Mockito.mock(MessageAndMetadata.class); + final MessageAndMetadata messageAndMetadata2 = Mockito.mock(MessageAndMetadata.class); + final MessageAndMetadata messageAndMetadata3 = Mockito.mock(MessageAndMetadata.class); + + Mockito.when(iterator1.next()).thenReturn(messageAndMetadata1); + Mockito.when(iterator2.next()).thenReturn(messageAndMetadata2); + Mockito.when(iterator3.next()).thenReturn(messageAndMetadata3); + + Mockito.when(messageAndMetadata1.message()).thenReturn("got message"); + Mockito.when(messageAndMetadata1.topic()).thenReturn("topic"); + Mockito.when(messageAndMetadata1.partition()).thenReturn(1); + + Mockito.when(messageAndMetadata2.message()).thenReturn("got message"); + Mockito.when(messageAndMetadata2.topic()).thenReturn("topic"); + Mockito.when(messageAndMetadata2.partition()).thenReturn(2); + + Mockito.when(messageAndMetadata3.message()).thenReturn("got message"); + Mockito.when(messageAndMetadata3.topic()).thenReturn("topic"); + Mockito.when(messageAndMetadata3.partition()).thenReturn(3); + + final Map>> messages = consumerConfiguration.receive(); + Assert.assertEquals(messages.size(), 1); + int sum = 0; + + final Map> values = messages.get("topic"); + + for (final List l : values.values()) { + sum += l.size(); + } + + Assert.assertEquals(sum, 3); + } + + @Test + @SuppressWarnings("unchecked") + public void testReceiveMessageForMultipleTopicsFromMultipleStreams() { + final ConsumerMetadata consumerMetadata = Mockito.mock(ConsumerMetadata.class); + final ConsumerConnectionProvider consumerConnectionProvider = + Mockito.mock(ConsumerConnectionProvider.class); + final MessageLeftOverTracker messageLeftOverTracker = Mockito.mock(MessageLeftOverTracker.class); + + final ConsumerConnector consumerConnector = Mockito.mock(ConsumerConnector.class); + + Mockito.when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector); + + final ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(consumerMetadata, + consumerConnectionProvider, messageLeftOverTracker); + consumerConfiguration.setMaxMessages(9); + + final KafkaStream stream1 = Mockito.mock(KafkaStream.class); + final KafkaStream stream2 = Mockito.mock(KafkaStream.class); + final KafkaStream stream3 = Mockito.mock(KafkaStream.class); + final List> streams = new ArrayList>(); + streams.add(stream1); + streams.add(stream2); + streams.add(stream3); + final Map>> messageStreams = new HashMap>>(); + messageStreams.put("topic1", streams); + messageStreams.put("topic2", streams); + messageStreams.put("topic3", streams); + + Mockito.when(consumerConfiguration.getConsumerMapWithMessageStreams()).thenReturn(messageStreams); + final ConsumerIterator iterator1 = Mockito.mock(ConsumerIterator.class); + final ConsumerIterator iterator2 = Mockito.mock(ConsumerIterator.class); + final ConsumerIterator iterator3 = Mockito.mock(ConsumerIterator.class); + + Mockito.when(stream1.iterator()).thenReturn(iterator1); + Mockito.when(stream2.iterator()).thenReturn(iterator2); + Mockito.when(stream3.iterator()).thenReturn(iterator3); + final MessageAndMetadata messageAndMetadata1 = Mockito.mock(MessageAndMetadata.class); + final MessageAndMetadata messageAndMetadata2 = Mockito.mock(MessageAndMetadata.class); + final MessageAndMetadata messageAndMetadata3 = Mockito.mock(MessageAndMetadata.class); + + Mockito.when(iterator1.next()).thenReturn(messageAndMetadata1); + Mockito.when(iterator2.next()).thenReturn(messageAndMetadata2); + Mockito.when(iterator3.next()).thenReturn(messageAndMetadata3); + + Mockito.when(messageAndMetadata1.message()).thenReturn("got message1"); + Mockito.when(messageAndMetadata1.topic()).thenReturn("topic1"); + Mockito.when(messageAndMetadata1.partition()).thenAnswer(getAnswer()); + + Mockito.when(messageAndMetadata2.message()).thenReturn("got message2"); + Mockito.when(messageAndMetadata2.topic()).thenReturn("topic2"); + Mockito.when(messageAndMetadata1.partition()).thenAnswer(getAnswer()); + + Mockito.when(messageAndMetadata3.message()).thenReturn("got message3"); + Mockito.when(messageAndMetadata3.topic()).thenReturn("topic3"); + Mockito.when(messageAndMetadata1.partition()).thenAnswer(getAnswer()); + + final Map>> messages = consumerConfiguration.receive(); + int sum = 0; + + final Collection>> values = messages.values(); + + for (final Map> m : values) { + for (final List l : m.values()) { + sum += l.size(); + } + } + + Assert.assertEquals(sum, 9); + } + + private Answer getAnswer() { + return new Answer() { + private int count = 0; + + @Override + public Object answer(final InvocationOnMock invocation) throws Throwable { + if (count++ == 1) { + return 1; + } else if (count++ == 2) { + return 2; + } + + return 3; + } + }; + } + + @Test + @SuppressWarnings("unchecked") + public void testReceiveMessageAndVerifyMessageLeftoverFromPreviousPollAreTakenFirst() { + final ConsumerMetadata consumerMetadata = Mockito.mock(ConsumerMetadata.class); + final ConsumerConnectionProvider consumerConnectionProvider = + Mockito.mock(ConsumerConnectionProvider.class); + final MessageLeftOverTracker messageLeftOverTracker = Mockito.mock(MessageLeftOverTracker.class); + final ConsumerConnector consumerConnector = Mockito.mock(ConsumerConnector.class); + + Mockito.when(messageLeftOverTracker.getCurrentCount()).thenReturn(3); + final MessageAndMetadata m1 = new MessageAndMetadata("key1", "value1", "topic1", 1, 1L); + final MessageAndMetadata m2 = new MessageAndMetadata("key2", "value2", "topic2", 1, 1L); + final MessageAndMetadata m3 = new MessageAndMetadata("key1", "value3", "topic3", 1, 1L); + + final List mList = new ArrayList(); + mList.add(m1); + mList.add(m2); + mList.add(m3); + + Mockito.when(messageLeftOverTracker.getMessageLeftOverFromPreviousPoll()).thenReturn(mList); + + Mockito.when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector); + + final ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(consumerMetadata, + consumerConnectionProvider, messageLeftOverTracker); + consumerConfiguration.setMaxMessages(5); + + final KafkaStream stream = Mockito.mock(KafkaStream.class); + final List> streams = new ArrayList>(); + streams.add(stream); + final Map>> messageStreams = new HashMap>>(); + messageStreams.put("topic1", streams); + + Mockito.when(consumerConfiguration.getConsumerMapWithMessageStreams()).thenReturn(messageStreams); + final ConsumerIterator iterator = Mockito.mock(ConsumerIterator.class); + Mockito.when(stream.iterator()).thenReturn(iterator); + final MessageAndMetadata messageAndMetadata = Mockito.mock(MessageAndMetadata.class); + Mockito.when(iterator.next()).thenReturn(messageAndMetadata); + Mockito.when(messageAndMetadata.message()).thenReturn("got message"); + Mockito.when(messageAndMetadata.topic()).thenReturn("topic1"); + Mockito.when(messageAndMetadata.partition()).thenReturn(1); + + final Map>> messages = consumerConfiguration.receive(); + int sum = 0; + + final Collection>> values = messages.values(); + + for (final Map> m : values) { + for (final List l : m.values()) { + sum += l.size(); + } + + } + Assert.assertEquals(sum, 5); + + Assert.assertTrue(messages.containsKey("topic1")); + Assert.assertTrue(messages.containsKey("topic2")); + Assert.assertTrue(messages.containsKey("topic3")); + + Assert.assertTrue(valueFound(messages.get("topic1").get(1), "value1")); + Assert.assertTrue(valueFound(messages.get("topic2").get(1), "value2")); + Assert.assertTrue(valueFound(messages.get("topic3").get(1), "value3")); + } + + private boolean valueFound(final List l, final String value){ + for (final Object o : l){ + if (value.equals(o)){ + return true; + } + } + + return false; + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaConsumerContextTest.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaConsumerContextTest.java new file mode 100644 index 0000000000..12bc189b2a --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaConsumerContextTest.java @@ -0,0 +1,80 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.ListableBeanFactory; +import org.springframework.integration.Message; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author Soby Chacko + */ +public class KafkaConsumerContextTest { + + @Test + public void testMergeResultsFromMultipleConsumerConfiguration() { + final KafkaConsumerContext kafkaConsumerContext = new KafkaConsumerContext(); + final ListableBeanFactory beanFactory = Mockito.mock(ListableBeanFactory.class); + final ConsumerConfiguration consumerConfiguration1 = Mockito.mock(ConsumerConfiguration.class); + final ConsumerConfiguration consumerConfiguration2 = Mockito.mock(ConsumerConfiguration.class); + + final Map map = new HashMap(); + map.put("config1", consumerConfiguration1); + map.put("config2", consumerConfiguration2); + + Mockito.when(beanFactory.getBeansOfType(ConsumerConfiguration.class)).thenReturn(map); + kafkaConsumerContext.setBeanFactory(beanFactory); + + final Map>> result1 = new HashMap>>(); + final List l1 = new ArrayList(); + l1.add("got message1 - l1"); + l1.add("got message2 - l1"); + final Map> innerMap1 = new HashMap>(); + innerMap1.put(1, l1); + result1.put("topic1", innerMap1); + + final Map>> result2 = new HashMap>>(); + final List l2 = new ArrayList(); + l2.add("got message1 - l2"); + l2.add("got message2 - l2"); + l2.add("got message3 - l2"); + + final Map> innerMap2 = new HashMap>(); + innerMap2.put(1, l2); + result1.put("topic2", innerMap2); + + Mockito.when(consumerConfiguration1.receive()).thenReturn(result1); + Mockito.when(consumerConfiguration2.receive()).thenReturn(result2); + + final Message>>> messages = kafkaConsumerContext.receive(); + Assert.assertEquals(messages.getPayload().size(), 2); + Assert.assertEquals(messages.getPayload().get("topic1").size(), 1); + Assert.assertEquals(messages.getPayload().get("topic1").get(1).get(0), "got message1 - l1"); + Assert.assertEquals(messages.getPayload().get("topic1").get(1).get(1), "got message2 - l1"); + + Assert.assertEquals(messages.getPayload().get("topic2").size(), 1); + Assert.assertEquals(messages.getPayload().get("topic2").get(1).get(0), "got message1 - l2"); + Assert.assertEquals(messages.getPayload().get("topic2").get(1).get(1), "got message2 - l2"); + Assert.assertEquals(messages.getPayload().get("topic2").get(1).get(2), "got message3 - l2"); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerConfigurationTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerConfigurationTests.java new file mode 100644 index 0000000000..2fa20219fa --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerConfigurationTests.java @@ -0,0 +1,303 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import kafka.javaapi.producer.Producer; +import kafka.producer.KeyedMessage; +import kafka.serializer.DefaultEncoder; +import kafka.serializer.StringEncoder; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.springframework.integration.Message; +import org.springframework.integration.kafka.serializer.avro.AvroBackedKafkaEncoder; +import org.springframework.integration.kafka.test.utils.NonSerializableTestKey; +import org.springframework.integration.kafka.test.utils.NonSerializableTestPayload; +import org.springframework.integration.kafka.test.utils.TestKey; +import org.springframework.integration.kafka.test.utils.TestPayload; +import org.springframework.integration.support.MessageBuilder; + +import java.io.ByteArrayInputStream; +import java.io.NotSerializableException; +import java.io.ObjectInputStream; + +/** + * @author Soby Chacko + */ +public class ProducerConfigurationTests { + @Test + @SuppressWarnings("unchecked") + public void testSendMessageWithNonDefaultKeyAndValueEncoders() throws Exception { + final ProducerMetadata producerMetadata = new ProducerMetadata("test"); + producerMetadata.setValueEncoder(new StringEncoder(null)); + producerMetadata.setKeyEncoder(new StringEncoder(null)); + producerMetadata.setKeyClassType(String.class); + producerMetadata.setValueClassType(String.class); + final Producer producer = Mockito.mock(Producer.class); + + final ProducerConfiguration configuration = new ProducerConfiguration(producerMetadata, producer); + + final Message message = MessageBuilder.withPayload("test message"). + setHeader("messageKey", "key") + .setHeader("topic", "test").build(); + + configuration.send(message); + + Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(KeyedMessage.class)); + + final ArgumentCaptor argument = ArgumentCaptor.forClass(KeyedMessage.class); + Mockito.verify(producer).send(argument.capture()); + + final KeyedMessage capturedKeyMessage = argument.getValue(); + + Assert.assertEquals(capturedKeyMessage.key(), "key"); + Assert.assertEquals(capturedKeyMessage.message(), "test message"); + Assert.assertEquals(capturedKeyMessage.topic(), "test"); + } + + /** + * User does not set an explicit key/value encoder, but send a serializable object for both key/value + */ + @Test + @SuppressWarnings("unchecked") + public void testSendMessageWithDefaultKeyAndValueEncodersAndCustomSerializableKeyAndPayloadObject() throws Exception { + final ProducerMetadata producerMetadata = new ProducerMetadata("test"); + producerMetadata.setValueEncoder(new DefaultEncoder(null)); + producerMetadata.setKeyEncoder(new DefaultEncoder(null)); + final Producer producer = Mockito.mock(Producer.class); + + final ProducerConfiguration configuration = new ProducerConfiguration(producerMetadata, producer); + + final Message message = MessageBuilder.withPayload(new TestPayload("part1", "part2")). + setHeader("messageKey", new TestKey("compositePart1", "compositePart2")) + .setHeader("topic", "test").build(); + + configuration.send(message); + + Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(KeyedMessage.class)); + + final ArgumentCaptor argument = ArgumentCaptor.forClass(KeyedMessage.class); + Mockito.verify(producer).send(argument.capture()); + + final KeyedMessage capturedKeyMessage = argument.getValue(); + + final byte[] keyBytes = (byte[])capturedKeyMessage.key(); + + final ByteArrayInputStream keyInputStream = new ByteArrayInputStream (keyBytes); + final ObjectInputStream keyObjectInputStream = new ObjectInputStream (keyInputStream); + final Object keyObj = keyObjectInputStream.readObject(); + + final TestKey tk = (TestKey)keyObj; + + Assert.assertEquals(tk.getKeyPart1(), "compositePart1"); + Assert.assertEquals(tk.getKeyPart2(), "compositePart2"); + + final byte[] messageBytes = (byte[])capturedKeyMessage.message(); + + final ByteArrayInputStream messageInputStream = new ByteArrayInputStream (messageBytes); + final ObjectInputStream messageObjectInputStream = new ObjectInputStream (messageInputStream); + final Object messageObj = messageObjectInputStream.readObject(); + + final TestPayload tp = (TestPayload)messageObj; + + Assert.assertEquals(tp.getPart1(), "part1"); + Assert.assertEquals(tp.getPart2(), "part2"); + + Assert.assertEquals(capturedKeyMessage.topic(), "test"); + } + + /** + * User does not set an explicit key encoder, but a value encoder, and sends the corresponding data + */ + @Test + @SuppressWarnings("unchecked") + public void testSendMessageWithDefaultKeyEncoderAndNonDefaultValueEncoderAndCorrespondingData() throws Exception { + final ProducerMetadata producerMetadata = new ProducerMetadata("test"); + final AvroBackedKafkaEncoder encoder = new AvroBackedKafkaEncoder(TestPayload.class); + producerMetadata.setValueEncoder(encoder); + producerMetadata.setKeyEncoder(new DefaultEncoder(null)); + producerMetadata.setValueClassType(TestPayload.class); + final Producer producer = Mockito.mock(Producer.class); + + final ProducerConfiguration configuration = new ProducerConfiguration(producerMetadata, producer); + final TestPayload tp = new TestPayload("part1", "part2"); + final Message message = MessageBuilder.withPayload(tp). + setHeader("messageKey", "key") + .setHeader("topic", "test").build(); + + configuration.send(message); + + Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(KeyedMessage.class)); + + final ArgumentCaptor argument = ArgumentCaptor.forClass(KeyedMessage.class); + Mockito.verify(producer).send(argument.capture()); + + final KeyedMessage capturedKeyMessage = argument.getValue(); + + final byte[] keyBytes = (byte[])capturedKeyMessage.key(); + + final ByteArrayInputStream keyInputStream = new ByteArrayInputStream (keyBytes); + final ObjectInputStream keyObjectInputStream = new ObjectInputStream (keyInputStream); + final Object keyObj = keyObjectInputStream.readObject(); + + Assert.assertEquals("key", keyObj); + Assert.assertEquals(capturedKeyMessage.message(), tp); + + Assert.assertEquals(capturedKeyMessage.topic(), "test"); + } + + /** + * User does set an explicit key encoder, but not a value encoder, and sends the corresponding data + */ + @Test + @SuppressWarnings("unchecked") + public void testSendMessageWithNonDefaultKeyEncoderAndDefaultValueEncoderAndCorrespondingData() throws Exception { + final ProducerMetadata producerMetadata = new ProducerMetadata("test"); + final AvroBackedKafkaEncoder encoder = new AvroBackedKafkaEncoder(TestKey.class); + producerMetadata.setKeyEncoder(encoder); + producerMetadata.setValueEncoder(new DefaultEncoder(null)); + producerMetadata.setKeyClassType(TestKey.class); + final Producer producer = Mockito.mock(Producer.class); + + final ProducerConfiguration configuration = new ProducerConfiguration(producerMetadata, producer); + final TestKey tk = new TestKey("part1", "part2"); + final Message message = MessageBuilder.withPayload("test message"). + setHeader("messageKey", tk) + .setHeader("topic", "test").build(); + + configuration.send(message); + + Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(KeyedMessage.class)); + + final ArgumentCaptor argument = ArgumentCaptor.forClass(KeyedMessage.class); + Mockito.verify(producer).send(argument.capture()); + + final KeyedMessage capturedKeyMessage = argument.getValue(); + + Assert.assertEquals(capturedKeyMessage.key(), tk); + + final byte[] payloadBytes = (byte[])capturedKeyMessage.message(); + + final ByteArrayInputStream payloadBis = new ByteArrayInputStream (payloadBytes); + final ObjectInputStream payloadOis = new ObjectInputStream (payloadBis); + final Object payloadObj = payloadOis.readObject(); + + Assert.assertEquals("test message", payloadObj); + + Assert.assertEquals(capturedKeyMessage.topic(), "test"); + } + + /** + * User does not set an explicit key/value encoder, but send a serializable String key/value pair + */ + @Test + @SuppressWarnings("unchecked") + public void testSendMessageWithDefaultKeyAndValueEncodersAndStringKeyAndValue() throws Exception { + final ProducerMetadata producerMetadata = new ProducerMetadata("test"); + producerMetadata.setValueEncoder(new DefaultEncoder(null)); + producerMetadata.setKeyEncoder(new DefaultEncoder(null)); + final Producer producer = Mockito.mock(Producer.class); + + final ProducerConfiguration configuration = new ProducerConfiguration(producerMetadata, producer); + + final Message message = MessageBuilder.withPayload("test message"). + setHeader("messageKey", "key") + .setHeader("topic", "test").build(); + + configuration.send(message); + + Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(KeyedMessage.class)); + + final ArgumentCaptor argument = ArgumentCaptor.forClass(KeyedMessage.class); + Mockito.verify(producer).send(argument.capture()); + + final KeyedMessage capturedKeyMessage = argument.getValue(); + final byte[] keyBytes = (byte[])capturedKeyMessage.key(); + + final ByteArrayInputStream keyBis = new ByteArrayInputStream (keyBytes); + final ObjectInputStream keyOis = new ObjectInputStream (keyBis); + final Object keyObj = keyOis.readObject(); + + Assert.assertEquals("key", keyObj); + + final byte[] payloadBytes = (byte[])capturedKeyMessage.message(); + + final ByteArrayInputStream payloadBis = new ByteArrayInputStream (payloadBytes); + final ObjectInputStream payloadOis = new ObjectInputStream (payloadBis); + final Object payloadObj = payloadOis.readObject(); + + Assert.assertEquals("test message", payloadObj); + Assert.assertEquals(capturedKeyMessage.topic(), "test"); + } + + /** + * User does not set an explicit key/value encoder, but send non-serializable object for both key/value + */ + @Test(expected = NotSerializableException.class) + @SuppressWarnings("unchecked") + public void testSendMessageWithDefaultKeyAndValueEncodersButNonSerializableKeyAndValue() throws Exception { + final ProducerMetadata producerMetadata = new ProducerMetadata("test"); + producerMetadata.setValueEncoder(new DefaultEncoder(null)); + producerMetadata.setKeyEncoder(new DefaultEncoder(null)); + final Producer producer = Mockito.mock(Producer.class); + + final ProducerConfiguration configuration = new ProducerConfiguration(producerMetadata, producer); + + final Message message = MessageBuilder.withPayload(new NonSerializableTestPayload("part1", "part2")). + setHeader("messageKey", new NonSerializableTestKey("compositePart1", "compositePart2")) + .setHeader("topic", "test").build(); + configuration.send(message); + } + + /** + * User does not set an explicit key/value encoder, but send non-serializable key and serializable value + */ + @Test(expected = NotSerializableException.class) + @SuppressWarnings("unchecked") + public void testSendMessageWithDefaultKeyAndValueEncodersButNonSerializableKeyAndSerializableValue() throws Exception { + final ProducerMetadata producerMetadata = new ProducerMetadata("test"); + producerMetadata.setValueEncoder(new DefaultEncoder(null)); + producerMetadata.setKeyEncoder(new DefaultEncoder(null)); + final Producer producer = Mockito.mock(Producer.class); + + final ProducerConfiguration configuration = new ProducerConfiguration(producerMetadata, producer); + + final Message message = MessageBuilder.withPayload(new TestPayload("part1", "part2")). + setHeader("messageKey", new NonSerializableTestKey("compositePart1", "compositePart2")) + .setHeader("topic", "test").build(); + configuration.send(message); + } + + /** + * User does not set an explicit key/value encoder, but send serializable key and non-serializable value + */ + @Test(expected = NotSerializableException.class) + @SuppressWarnings("unchecked") + public void testSendMessageWithDefaultKeyAndValueEncodersButSerializableKeyAndNonSerializableValue() throws Exception { + final ProducerMetadata producerMetadata = new ProducerMetadata("test"); + producerMetadata.setValueEncoder(new DefaultEncoder(null)); + producerMetadata.setKeyEncoder(new DefaultEncoder(null)); + final Producer producer = Mockito.mock(Producer.class); + + final ProducerConfiguration configuration = new ProducerConfiguration(producerMetadata, producer); + + final Message message = MessageBuilder.withPayload(new NonSerializableTestPayload("part1", "part2")). + setHeader("messageKey", new TestKey("compositePart1", "compositePart2")) + .setHeader("topic", "test").build(); + configuration.send(message); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerFactoryBeanTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerFactoryBeanTests.java new file mode 100644 index 0000000000..0089e73a8d --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerFactoryBeanTests.java @@ -0,0 +1,63 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.support; + +import junit.framework.Assert; +import kafka.javaapi.producer.Producer; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * @author Soby Chacko + */ +public class ProducerFactoryBeanTests { + + @Test + public void createProducerWithDefaultMetadata() throws Exception { + final ProducerMetadata producerMetadata = new ProducerMetadata("test"); + final ProducerMetadata tm = Mockito.spy(producerMetadata); + final ProducerFactoryBean producerFactoryBean = new ProducerFactoryBean(tm, "localhost:9092"); + final Producer producer = producerFactoryBean.getObject(); + + Assert.assertTrue(producer != null); + + Mockito.verify(tm, Mockito.times(1)).getPartitioner(); + Mockito.verify(tm, Mockito.times(1)).getCompressionCodec(); + Mockito.verify(tm, Mockito.times(1)).getValueEncoder(); + Mockito.verify(tm, Mockito.times(1)).getKeyEncoder(); + Mockito.verify(tm, Mockito.times(1)).isAsync(); + Mockito.verify(tm, Mockito.times(0)).getBatchNumMessages(); + } + + @Test + public void createProducerWithAsyncFeatures() throws Exception { + final ProducerMetadata producerMetadata = new ProducerMetadata("test"); + producerMetadata.setAsync(true); + producerMetadata.setBatchNumMessages("300"); + final ProducerMetadata tm = Mockito.spy(producerMetadata); + final ProducerFactoryBean producerFactoryBean = new ProducerFactoryBean(tm, "localhost:9092"); + final Producer producer = producerFactoryBean.getObject(); + + Assert.assertTrue(producer != null); + + Mockito.verify(tm, Mockito.times(1)).getPartitioner(); + Mockito.verify(tm, Mockito.times(1)).getCompressionCodec(); + Mockito.verify(tm, Mockito.times(1)).getValueEncoder(); + Mockito.verify(tm, Mockito.times(1)).getKeyEncoder(); + Mockito.verify(tm, Mockito.times(1)).isAsync(); + Mockito.verify(tm, Mockito.times(2)).getBatchNumMessages(); + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/NonSerializableTestKey.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/NonSerializableTestKey.java new file mode 100644 index 0000000000..eef22dfaa8 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/NonSerializableTestKey.java @@ -0,0 +1,35 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.test.utils; + +public class NonSerializableTestKey { + private final String keyPart1; + private final String keyPart2; + + public NonSerializableTestKey(final String keyPart1, final String keyPart2) { + this.keyPart1 = keyPart1; + this.keyPart2 = keyPart2; + } + + public String getKeyPart1() { + return keyPart1; + } + + public String getKeyPart2() { + return keyPart2; + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/NonSerializableTestPayload.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/NonSerializableTestPayload.java new file mode 100644 index 0000000000..6b2bb2f9ce --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/NonSerializableTestPayload.java @@ -0,0 +1,34 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.test.utils; + +public class NonSerializableTestPayload { + private final String part1; + private final String part2; + + public NonSerializableTestPayload(final String part1, final String part2) { + this.part1 = part1; + this.part2 = part2; + } + + public String getPart1() { + return part1; + } + + public String getPart2() { + return part2; + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestKey.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestKey.java new file mode 100644 index 0000000000..10e04801fc --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestKey.java @@ -0,0 +1,38 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.test.utils; + +import java.io.Serializable; + +public class TestKey implements Serializable { + private static final long serialVersionUID = -6415387283545560656L; + + private final String keyPart1; + private final String keyPart2; + + public TestKey(final String keyPart1, final String keyPart2) { + this.keyPart1 = keyPart1; + this.keyPart2 = keyPart2; + } + + public String getKeyPart1() { + return keyPart1; + } + + public String getKeyPart2() { + return keyPart2; + } +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestObject.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestObject.java new file mode 100644 index 0000000000..42c2ec15b2 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestObject.java @@ -0,0 +1,38 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.test.utils; + +public class TestObject { + public String testData1; + public int testData2; + + public String getTestData1() { + return testData1; + } + + public void setTestData1(final String testData1) { + this.testData1 = testData1; + } + + public int getTestData2() { + return testData2; + } + + public void setTestData2(final int testData2) { + this.testData2 = testData2; + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestPayload.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestPayload.java new file mode 100644 index 0000000000..3ac94f8f07 --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/TestPayload.java @@ -0,0 +1,38 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.integration.kafka.test.utils; + +import java.io.Serializable; + +public class TestPayload implements Serializable { + private static final long serialVersionUID = -8560378224929007403L; + + private final String part1; + private final String part2; + + public TestPayload(final String part1, final String part2){ + this.part1 = part1; + this.part2 = part2; + } + + public String getPart1() { + return part1; + } + + public String getPart2() { + return part2; + } +} diff --git a/spring-integration-kafka/src/test/resources/log4j.properties b/spring-integration-kafka/src/test/resources/log4j.properties new file mode 100644 index 0000000000..f428439196 --- /dev/null +++ b/spring-integration-kafka/src/test/resources/log4j.properties @@ -0,0 +1,8 @@ +log4j.rootCategory=WARN, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{HH:mm:ss.SSS} %-5p [%t][%c] %m%n + +log4j.category.org.springframework.integration=WARN +log4j.category.org.springframework.integration.kafka=INFO diff --git a/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaConsumerContextParserTests-context.xml b/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaConsumerContextParserTests-context.xml new file mode 100644 index 0000000000..e0978bbf19 --- /dev/null +++ b/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaConsumerContextParserTests-context.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + diff --git a/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterCommon-context.xml b/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterCommon-context.xml new file mode 100644 index 0000000000..f7fecae4b0 --- /dev/null +++ b/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterCommon-context.xml @@ -0,0 +1,17 @@ + + + + + + + + + diff --git a/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterParserTests-context.xml b/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterParserTests-context.xml new file mode 100644 index 0000000000..70266d3684 --- /dev/null +++ b/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterParserTests-context.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaOutboundAdapterParserTests-context.xml b/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaOutboundAdapterParserTests-context.xml new file mode 100644 index 0000000000..52eef96071 --- /dev/null +++ b/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaOutboundAdapterParserTests-context.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaProducerContextParserTests-context.xml b/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaProducerContextParserTests-context.xml new file mode 100644 index 0000000000..8f9750af79 --- /dev/null +++ b/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/kafkaProducerContextParserTests-context.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + diff --git a/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/zookeeperConnectParserTests-context.xml b/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/zookeeperConnectParserTests-context.xml new file mode 100644 index 0000000000..f5d07846a6 --- /dev/null +++ b/spring-integration-kafka/src/test/resources/org/springframework/integration/kafka/config/xml/zookeeperConnectParserTests-context.xml @@ -0,0 +1,14 @@ + + + + + + +